scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-19 00:01:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 88

fix(proxy): estimate auto-router baseline costs from durable cache history
Some checks failed
ai-gateway image / ai-gateway release image (push) Has been cancelled
Details
LiteLLM Rust / rust-lint (push) Has been cancelled
Details
LiteLLM Rust / rust-test (push) Has been cancelled
Details

Browse source
This commit is contained in:
Tin Chi Lo 2026-09-14 20:16:05 -07:00
parent 3b14631f06
commit 64610c1ba8
65 changed files with 4312 additions and 651 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_add_autorouter_savings_estimate_coverage/migration.sql Normal file
View file

@ -0,0 +1,5 @@
ALTER TABLE "LiteLLM_AutoRouterSession"
ADD COLUMN IF NOT EXISTS "savings_estimated_turns" INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS "savings_estimated_actual_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS "savings_estimated_saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS "savings_estimated_baseline_models" JSONB NOT NULL DEFAULT '{}';

36
litellm-proxy-extras/litellm_proxy_extras/migrations/20260915010000_add_autorouter_baseline_state/migration.sql Normal file
View file

@ -0,0 +1,36 @@
CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterBaselineComparison" (
"scope" TEXT PRIMARY KEY,
"api_key" TEXT NOT NULL,
"session_id" TEXT NOT NULL,
"router_name" TEXT NOT NULL,
"initial_equivalent" BOOLEAN NOT NULL,
"revision" BIGINT NOT NULL DEFAULT 0,
"published_revision" BIGINT NOT NULL DEFAULT 0,
"history" TEXT,
"attempted_at" TIMESTAMP(3),
"retired" BOOLEAN NOT NULL DEFAULT FALSE,
"updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS "idx_autorouter_baseline_scope"
ON "LiteLLM_AutoRouterBaselineComparison" ("api_key", "session_id", "router_name");
CREATE INDEX IF NOT EXISTS "idx_autorouter_baseline_updated"
ON "LiteLLM_AutoRouterBaselineComparison" ("updated_at");
CREATE INDEX IF NOT EXISTS "idx_autorouter_baseline_dirty"
ON "LiteLLM_AutoRouterBaselineComparison" ("attempted_at", "updated_at", "scope")
WHERE NOT "retired" AND "revision" <> "published_revision";
CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterBaselineObservation" (
"request_id" TEXT PRIMARY KEY,
"scope" TEXT NOT NULL,
"started_at" DOUBLE PRECISION NOT NULL,
"revision" BIGINT NOT NULL,
"data" TEXT NOT NULL,
"publication" TEXT,
"conflicted" BOOLEAN NOT NULL DEFAULT FALSE
);
CREATE INDEX IF NOT EXISTS "idx_autorouter_baseline_event_order"
ON "LiteLLM_AutoRouterBaselineObservation" ("scope", "started_at", "request_id");
CREATE INDEX IF NOT EXISTS "idx_autorouter_baseline_event_revision"
ON "LiteLLM_AutoRouterBaselineObservation" ("scope", "revision", "started_at");

34
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -1508,6 +1508,36 @@ model LiteLLM_AdaptiveRouterSession {
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
}
model LiteLLM_AutoRouterBaselineComparison {
scope String @id
api_key String
session_id String
router_name String
initial_equivalent Boolean
revision BigInt @default(0)
published_revision BigInt @default(0)
history String?
attempted_at DateTime?
retired Boolean @default(false)
updated_at DateTime @default(now())
@@index([api_key, session_id, router_name], map: "idx_autorouter_baseline_scope")
@@index([updated_at], map: "idx_autorouter_baseline_updated")
}
model LiteLLM_AutoRouterBaselineObservation {
request_id String @id
scope String
started_at Float
revision BigInt
data String
publication String?
conflicted Boolean @default(false)
@@index([scope, started_at, request_id], map: "idx_autorouter_baseline_event_order")
@@index([scope, revision, started_at], map: "idx_autorouter_baseline_event_revision")
}
model LiteLLM_AutoRouterSession {
api_key String
session_id String
@ -1534,6 +1564,10 @@ model LiteLLM_AutoRouterSession {
total_tokens BigInt @default(0)
spend Float @default(0)
saved_spend Float @default(0)
savings_estimated_turns Int @default(0)
savings_estimated_actual_spend Float @default(0)
savings_estimated_saved_spend Float @default(0)
savings_estimated_baseline_models Json @default("{}")
classifier_cost Float @default(0)
classifier_cost_recorded_turns Int @default(0)
tier_turns Json @default("{}")

75
litellm/litellm_core_utils/litellm_logging.py
View file

@ -211,6 +211,7 @@ if TYPE_CHECKING:
from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
from litellm.litellm_core_utils.llm_cost_calc.utils import BilledTokenRates
from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector
from litellm.proxy.hooks.autorouter_baseline_cache import BaselineCacheContext, CapturedBaselineObservation
try:
from litellm_enterprise.enterprise_callbacks.callback_controls import (
EnterpriseCallbackControls,
@ -494,6 +495,8 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_request_debug: bool = False
streamed_anthropic_message_id: str | None = None
classifier_input: Mapping[str, JsonValue] | None = None
baseline_cache_context: "BaselineCacheContext | None" = None
baseline_observation: "CapturedBaselineObservation | None" = None
def __init__(
self,
@ -501,7 +504,7 @@ class Logging(LiteLLMLoggingBaseClass):
messages,
stream,
call_type,
start_time,
start_time: datetime.datetime,
litellm_call_id: str,
function_id: str,
litellm_trace_id: str | None = None,
@ -2200,6 +2203,19 @@ class Logging(LiteLLMLoggingBaseClass):
if standard_logging_payload is not None:
emit_standard_logging_payload(standard_logging_payload)
async def _prepare_baseline_cache_estimate(self, response_obj: object) -> None:
if self.baseline_cache_context is None:
return
from litellm.proxy.hooks.autorouter_baseline_cache import finalize_baseline_cache
await finalize_baseline_cache(self, response_obj)
async def invalidate_baseline_cache_estimate(self, reason: str, *, completed: bool = False) -> None:
"""Invalidate uncertain attempts; retire the reservation at logical completion."""
from litellm.proxy.hooks.autorouter_baseline_cache import invalidate_baseline_cache
await invalidate_baseline_cache(self, reason, completed=completed)
def _build_standard_logging_payload(
self, init_response_obj: object, start_time: Any, end_time: Any
) -> StandardLoggingPayload | None:
@ -3038,8 +3054,17 @@ class Logging(LiteLLMLoggingBaseClass):
result=result,
cache_hit=cache_hit,
standard_logging_object=kwargs.get("standard_logging_object", None),
build_logging_payload=self.baseline_cache_context is None,
)
if self.stream is not True and self.baseline_cache_context is not None:
await self._prepare_baseline_cache_estimate(result)
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
result, start_time, end_time
)
if (prepared_payload := self.model_call_details.get("standard_logging_object")) is not None:
emit_standard_logging_payload(prepared_payload)
## BUILD COMPLETE STREAMED RESPONSE
if "async_complete_streaming_response" in self.model_call_details:
return # break out of this.
@ -3084,6 +3109,8 @@ class Logging(LiteLLMLoggingBaseClass):
self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
await self._prepare_baseline_cache_estimate(complete_streaming_response)
## STANDARDIZED LOGGING PAYLOAD
try:
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
@ -3112,6 +3139,7 @@ class Logging(LiteLLMLoggingBaseClass):
# Only build standard_logging_object if not already built by
# _success_handler_helper_fn
if self.model_call_details.get("standard_logging_object") is None:
await self._prepare_baseline_cache_estimate(result)
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
result, start_time, end_time
@ -3623,6 +3651,8 @@ class Logging(LiteLLMLoggingBaseClass):
"""
Implementing async callbacks, to handle asyncio event loop issues when custom integrations need to use async functions.
"""
if self.baseline_cache_context is not None:
await self.invalidate_baseline_cache_estimate("failed_request")
await self.special_failure_handlers(exception=exception)
if not self.should_run_logging(event_type="async_failure"): # prevent double logging
return
@ -6130,6 +6160,8 @@ def _autorouter_savings_for_payload(
model_id: str | None,
usage_object: Mapping[str, object] | None,
cost_breakdown: Mapping[str, object] | None,
baseline_usage: Usage | None = None,
baseline_provenance: Literal["observed_initial", "modeled"] | None = None,
) -> float | None:
"""The auto-router savings figure for the payload, or ``None`` when there is none.
@ -6148,6 +6180,8 @@ def _autorouter_savings_for_payload(
model_id=model_id,
usage_object=usage_object,
cost_breakdown=cost_breakdown,
baseline_usage=baseline_usage,
baseline_provenance=baseline_provenance,
)
except Exception as e: # noqa: BLE001 # a savings figure must never fail request logging
verbose_logger.debug("autorouter savings skipped on logging payload: %s", e)
@ -6324,13 +6358,18 @@ def get_standard_logging_object_payload(
model_name = response_model_name
request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
autorouter_savings: Final = _autorouter_savings_for_payload(
request_metadata=metadata,
model=model_name,
custom_llm_provider=custom_llm_provider,
model_id=_model_id,
usage_object=usage_dict,
cost_breakdown=request_cost_breakdown,
captured_baseline: Final = logging_obj.baseline_observation
autorouter_savings: Final = (
None
if status != "success" or cache_hit or logging_obj.baseline_cache_context is not None
else _autorouter_savings_for_payload(
request_metadata=metadata,
model=model_name,
custom_llm_provider=custom_llm_provider,
model_id=_model_id,
usage_object=usage_dict,
cost_breakdown=request_cost_breakdown,
)
)
payload: Final[StandardLoggingPayload] = StandardLoggingPayload(
@ -6377,6 +6416,26 @@ def get_standard_logging_object_payload(
response_cost=response_cost,
cost_breakdown=request_cost_breakdown,
autorouter_savings=autorouter_savings,
autorouter_savings_estimate=(
{
"version": 3,
"status": "unknown",
"reason": "pending_projection",
} # mutable-ok: spend-log JSON serialization requires plain mappings
if captured_baseline is not None
else (
{ # mutable-ok: spend-log JSON serialization requires plain mappings
"version": 1,
"status": "estimated" if autorouter_savings is not None else "unknown",
"reason": "uncached_usage" if autorouter_savings is not None else "baseline_unavailable",
}
if metadata.get("routing_decision")
else None
)
),
autorouter_baseline_observation=(
captured_baseline.model_dump_json() if captured_baseline is not None else None
),
total_tokens=usage_dict.get("total_tokens", 0),
prompt_tokens=usage_dict.get("prompt_tokens", 0),
completion_tokens=usage_dict.get("completion_tokens", 0),

11
litellm/llms/anthropic/chat/handler.py
View file

@ -4,7 +4,8 @@ Calling + translation logic for anthropic's `/v1/messages` endpoint
import copy
import json
from collections.abc import Callable
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Union, cast
import httpx
@ -33,7 +34,6 @@ from litellm.types.llms.anthropic import (
ContentBlockStop,
MessageBlockDelta,
MessageStartBlock,
UsageDelta,
)
from litellm.types.llms.openai import (
ChatCompletionRedactedThinkingBlock,
@ -623,6 +623,7 @@ class ModelResponseIterator:
self.tool_index = -1
self.json_mode = json_mode
self.speed = speed
self._cumulative_usage: Mapping[str, object] = MappingProxyType({})
# rewritten-name -> caller's original. Built per-request from the
# forward map in AnthropicConfig._build_request_tool_name_maps; only
# contains entries we actually rewrote, so a tool legitimately named
@ -696,10 +697,12 @@ class ModelResponseIterator:
return True
return False
def _handle_usage(self, anthropic_usage_chunk: dict | UsageDelta) -> Usage:
def _handle_usage(self, anthropic_usage_chunk: Mapping[str, object]) -> Usage:
# message_delta usage is cumulative but may omit fields reported at message_start.
self._cumulative_usage = MappingProxyType({**self._cumulative_usage, **anthropic_usage_chunk})
reasoning_content: Final = "".join(self.reasoning_content_chunks) if self.reasoning_content_chunks else None
usage: Final = AnthropicConfig().calculate_usage(
usage_object=cast(dict, anthropic_usage_chunk),
usage_object=self._cumulative_usage,
reasoning_content=reasoning_content,
speed=self.speed,
)

20
litellm/llms/anthropic/count_tokens/handler.py
View file

@ -4,9 +4,11 @@ Anthropic CountTokens API handler.
Uses httpx for HTTP requests instead of the Anthropic SDK.
"""
from typing import Any, Final
from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -16,6 +18,8 @@ from litellm.llms.anthropic.count_tokens.transformation import (
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
_COUNT_RESPONSE: Final = TypeAdapter(dict[str, JsonValue])
class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
"""
@ -27,13 +31,14 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
async def handle_count_tokens_request(
self,
model: str,
messages: list[dict[str, Any]],
messages: list[dict[str, JsonValue]],
api_key: str,
api_base: str | None = None,
timeout: float | httpx.Timeout | None = None,
tools: list[dict[str, Any]] | None = None,
system: Any | None = None,
) -> dict[str, Any]:
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
) -> dict[str, JsonValue]:
"""
Handle a CountTokens request using httpx.
@ -52,7 +57,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
"""
try:
# Validate the request
self.validate_request(model, messages)
self.validate_request(model, messages, system=system, tools=tools)
verbose_logger.debug("Processing Anthropic CountTokens request for model: %s", model)
@ -62,6 +67,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
messages=messages,
tools=tools,
system=system,
optional_params=optional_params,
)
verbose_logger.debug("Transformed request: %s", request_body)
@ -97,7 +103,7 @@ class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
message=error_text,
)
anthropic_response: Final = response.json()
anthropic_response: Final = _COUNT_RESPONSE.validate_json(response.content)
verbose_logger.debug("Anthropic response: %s", anthropic_response)

56
litellm/llms/anthropic/count_tokens/transformation.py
View file

@ -4,10 +4,17 @@ Anthropic CountTokens API transformation logic.
This module handles the transformation of requests to Anthropic's CountTokens API format.
"""
from typing import Any, Final
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
from pydantic import JsonValue, TypeAdapter
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
_COUNT_REQUEST: Final = TypeAdapter(dict[str, JsonValue])
COUNT_TOKEN_OPTION_NAMES: Final = ("thinking", "tool_choice", "output_config")
class AnthropicCountTokensConfig:
"""
@ -31,27 +38,31 @@ class AnthropicCountTokensConfig:
def transform_request_to_count_tokens(
self,
model: str,
messages: list[dict[str, Any]],
tools: list[dict[str, Any]] | None = None,
system: Any | None = None,
) -> dict[str, Any]:
messages: list[dict[str, JsonValue]],
tools: list[dict[str, JsonValue]] | None = None,
system: JsonValue = None,
optional_params: Mapping[str, JsonValue] | None = None,
) -> dict[str, JsonValue]: # mutable-ok: provider transport requires JSON dictionaries
"""
Transform request to Anthropic CountTokens format.
Includes optional system and tools fields for accurate token counting.
"""
request: Final[dict[str, Any]] = {
"model": model,
"messages": messages,
}
if system is not None:
request["system"] = system
if tools is not None:
request["tools"] = tools
return request
options: Final[Mapping[str, JsonValue]] = optional_params or MappingProxyType({})
return _COUNT_REQUEST.validate_python(
MappingProxyType(
{
"model": model,
"messages": messages,
**MappingProxyType(
{key: value for key, value in (("system", system), ("tools", tools)) if value is not None}
),
**MappingProxyType(
{key: value for key, value in options.items() if key in COUNT_TOKEN_OPTION_NAMES}
),
}
)
)
def get_required_headers(self, api_key: str) -> dict[str, str]:
"""
@ -76,7 +87,14 @@ class AnthropicCountTokensConfig:
headers, _ = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key)
return headers
def validate_request(self, model: str, messages: list[dict[str, Any]]) -> None:
def validate_request(
self,
model: str,
messages: Sequence[Mapping[str, JsonValue]],
*,
system: JsonValue = None,
tools: list[dict[str, JsonValue]] | None = None,
) -> None:
"""
Validate the incoming count tokens request.
@ -90,7 +108,7 @@ class AnthropicCountTokensConfig:
if not model:
raise ValueError("model parameter is required")
if not messages:
if not messages and not system and not tools:
raise ValueError("messages parameter is required")
if not isinstance(messages, list):

462
litellm/llms/anthropic/prompt_cache_prediction.py
View file

@ -1,10 +1,11 @@
from __future__ import annotations
import asyncio
import hashlib
import json
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from itertools import accumulate
from dataclasses import dataclass, field
from itertools import accumulate, groupby
from types import MappingProxyType
from typing import Annotated, Final, Literal, Protocol, TypeAlias
@ -14,9 +15,14 @@ from pydantic import BaseModel, ConfigDict, Field, JsonValue, StrictInt, TypeAda
import litellm
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, is_anthropic_oauth_key
from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import DEFAULT_ANTHROPIC_API_VERSION
from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
DEFAULT_ANTHROPIC_API_VERSION,
AnthropicMessagesConfig,
)
from litellm.types.router import LiteLLM_Params
from litellm.types.utils import ModelResponse
from litellm.utils import supports_thinking_cache_preservation
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_HEADERS: Final = TypeAdapter(dict[str, str])
@ -100,10 +106,7 @@ _Block: TypeAlias = Annotated[_Text | _ToolUse | _ToolResult, Field(discriminato
class _Message(_StrictModel):
role: Literal["user", "assistant"]
content: str | Annotated[tuple[_Block, ...], Field(strict=False)]
def blocks(self) -> tuple[_Text | _ToolUse | _ToolResult, ...]:
return (_Text(type="text", text=self.content),) if isinstance(self.content, str) else tuple(self.content)
content: Annotated[str, Field(min_length=1, pattern=r"\S")] | Annotated[tuple[_Block, ...], Field(strict=False)]
class _Tool(_StrictModel):
@ -113,10 +116,7 @@ class _Tool(_StrictModel):
type: Literal["custom"] | None = None
class _Request(_StrictModel):
messages: tuple[_Message, ...] = Field(min_length=1, strict=False)
system: str | Annotated[tuple[_ResultText, ...], Field(strict=False)] | None = None
tools: Annotated[tuple[_Tool, ...], Field(strict=False)] | None = None
class _RequestOptions(_StrictModel):
model: str | None = None
max_tokens: int | None = None
stream: bool | None = None
@ -127,6 +127,289 @@ class _Request(_StrictModel):
metadata: Mapping[str, JsonValue] | None = None
class _Request(_RequestOptions):
messages: tuple[_Message, ...] = Field(min_length=1, strict=False)
system: str | Annotated[tuple[_ResultText, ...], Field(strict=False)] | None = None
tools: Annotated[tuple[_Tool, ...], Field(strict=False)] | None = None
class _Thinking(_StrictModel):
type: Literal["thinking"]
thinking: str
signature: str = Field(min_length=1)
_PlanBlock: TypeAlias = Annotated[_Text | _ToolUse | _ToolResult | _Thinking, Field(discriminator="type")]
class _PlanMessage(_StrictModel):
role: Literal["user", "assistant", "system"]
content: str | Annotated[tuple[_PlanBlock, ...], Field(strict=False)]
class _PlanTool(_Tool):
cache_control: _CacheControl | None = None
class _PlanRequest(_RequestOptions):
messages: tuple[_PlanMessage, ...] = Field(min_length=1, strict=False)
system: str | Annotated[tuple[_Text, ...], Field(strict=False)] | None = None
tools: Annotated[tuple[_PlanTool, ...], Field(strict=False)] | None = None
cache_control: _CacheControl | None = None
thinking: Mapping[str, JsonValue] | None = None
tool_choice: Mapping[str, JsonValue] | None = None
output_config: Mapping[str, JsonValue] | None = None
speed: Literal["fast", "standard"] | None = None
service_tier: Literal["auto", "standard_only"] | None = None
@dataclass(frozen=True, slots=True)
class CacheBoundary:
fingerprint: str
prefix_body: Mapping[str, JsonValue] = field(repr=False)
ttl_seconds: int
lookback_fingerprints: tuple[str, ...]
content_fingerprint: str = ""
lookback_content_fingerprints: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
class PromptCachePlan:
full_body: Mapping[str, JsonValue] = field(repr=False)
breakpoints: tuple[CacheBoundary, ...]
@dataclass(frozen=True, slots=True)
class UnsupportedCachePlan:
reason: Literal[
"unsupported_prompt_shape",
"conflicting_cache_ttl",
"too_many_cache_breakpoints",
"invalid_cache_ttl_order",
"unsupported_thinking_cache_semantics",
"token_count_unavailable",
"inconsistent_prefix_token_count",
]
@dataclass(frozen=True, slots=True)
class CountedBreakpoint:
fingerprint: str
ttl_seconds: int
prefix_tokens: int
lookback_fingerprints: tuple[str, ...]
content_fingerprint: str = ""
lookback_content_fingerprints: tuple[str, ...] = ()
@dataclass(frozen=True, slots=True)
class CountedPromptCachePlan:
total_tokens: int
breakpoints: tuple[CountedBreakpoint, ...]
@dataclass(frozen=True, slots=True)
class _Position:
section: Literal["tools", "system", "messages"]
message_index: int
role: str
block: Mapping[str, JsonValue]
marker: _CacheControl | None
def _content_blocks(content: JsonValue) -> tuple[Mapping[str, JsonValue], ...]:
if isinstance(content, str):
return (MappingProxyType({"type": "text", "text": content}),)
return tuple(_JSON_OBJECT.validate_python(block) for block in content) if isinstance(content, list) else ()
def _position(
section: Literal["tools", "system", "messages"],
message_index: int,
role: str,
block: Mapping[str, JsonValue],
) -> _Position:
control: Final = block.get("cache_control")
return _Position(
section,
message_index,
role,
MappingProxyType({key: value for key, value in block.items() if key != "cache_control"}),
_CacheControl.model_validate(control) if control is not None else None,
)
def _positions(body: Mapping[str, JsonValue]) -> tuple[_Position, ...]:
tools: Final = body.get("tools")
messages: Final = body.get("messages")
return (
*tuple(
_position("tools", -1, "", _JSON_OBJECT.validate_python(tool))
for tool in (tools if isinstance(tools, list) else ())
),
*tuple(_position("system", -1, "", block) for block in _content_blocks(body.get("system"))),
*tuple(
_position("messages", message_index, str(message.get("role")), block)
for message_index, raw_message in enumerate(messages if isinstance(messages, list) else ())
for message in (_JSON_OBJECT.validate_python(raw_message),)
for block in _content_blocks(message.get("content"))
),
)
def _prefix_body(
body: Mapping[str, JsonValue],
positions: tuple[_Position, ...],
last_index: int,
) -> Mapping[str, JsonValue]:
prefix: Final = positions[: last_index + 1]
sections: Final = MappingProxyType(
{
section: _count_objects(tuple(position.block for position in prefix if position.section == section))
for section in ("tools", "system")
if any(position.section == section for position in prefix)
}
)
messages: Final = tuple(
MappingProxyType(
_JSON_OBJECT.validate_python(
MappingProxyType(
{"role": group[0].role, "content": _count_objects(tuple(position.block for position in group))}
)
)
)
for _, values in groupby(
(position for position in prefix if position.section == "messages"),
key=lambda position: position.message_index,
)
for group in (tuple(values),)
)
return MappingProxyType(
_JSON_OBJECT.validate_python(
MappingProxyType(
{
**MappingProxyType({key: body[key] for key in COUNT_TOKEN_OPTION_NAMES if key in body}),
**sections,
"messages": _count_objects(messages),
}
)
)
)
def _position_group(position: _Position, index: int) -> tuple[str, int, str | int]:
block_type: Final = position.block.get("type")
return (
position.section,
position.message_index,
block_type if isinstance(block_type, str) and block_type in ("tool_use", "tool_result") else index,
)
def _chain_digest(previous: str, current: str) -> str:
return _digest((previous, current))
def _cacheable_position(position: _Position) -> bool:
block_type: Final = position.block.get("type")
if block_type == "thinking":
return False
text: Final = position.block.get("text")
return block_type != "text" or (isinstance(text, str) and bool(text.strip()))
def _entry_fingerprint(fingerprint: str, ttl_seconds: int) -> str:
return _digest(("native-cache-prefix-v2", fingerprint, ttl_seconds))
def parse_cache_plan(body: Mapping[str, JsonValue]) -> PromptCachePlan | UnsupportedCachePlan:
try:
request: Final = _PlanRequest.model_validate(body)
positions: Final = _positions(body)
except ValidationError:
return UnsupportedCachePlan("unsupported_prompt_shape")
explicit: Final = tuple(
(index, position.marker) for index, position in enumerate(positions) if position.marker is not None
)
automatic_index: Final = next(
(index for index in reversed(range(len(positions))) if _cacheable_position(positions[index])), None
)
automatic_existing: Final = next((marker for index, marker in explicit if index == automatic_index), None)
if (
request.cache_control is not None
and automatic_existing is not None
and automatic_existing != request.cache_control
):
return UnsupportedCachePlan("conflicting_cache_ttl")
automatic: Final = (
((automatic_index, request.cache_control),)
if (request.cache_control is not None and automatic_index is not None and automatic_existing is None)
else ()
)
markers: Final = tuple(sorted((*explicit, *automatic), key=lambda value: value[0]))
if len(markers) > 4:
return UnsupportedCachePlan("too_many_cache_breakpoints")
ttls: Final = tuple(3600 if marker.ttl == "1h" else 300 for _, marker in markers)
if any(first < second for first, second in zip(ttls, ttls[1:])):
return UnsupportedCachePlan("invalid_cache_ttl_order")
settings: Final = MappingProxyType(
{
key: body[key]
for key in ("thinking", "output_config", "speed")
if key in body and not (key == "speed" and body[key] == "standard")
}
)
hashes: Final = tuple(
accumulate(
(
_digest(
(
position.section,
position.message_index,
position.role,
position.block,
body.get("tool_choice") if position.section == "messages" else None,
)
)
for position in positions
),
_chain_digest,
initial=_digest(settings),
)
)[1:]
groups: Final = tuple(
tuple(index for index, _ in values)
for _, values in groupby(
enumerate(positions),
key=lambda item: _position_group(item[1], item[0]),
)
)
return PromptCachePlan(
full_body=MappingProxyType(dict(body)),
breakpoints=tuple(
CacheBoundary(
fingerprint=_entry_fingerprint(hashes[index], ttl),
prefix_body=_prefix_body(body, positions, index),
ttl_seconds=ttl,
lookback_fingerprints=tuple(
_entry_fingerprint(hashes[earlier], ttl)
for group in reversed(tuple(group for group in groups if group[0] <= index)[-20:])
for earlier in reversed(group)
if earlier <= index
),
content_fingerprint=hashes[index],
lookback_content_fingerprints=tuple(
hashes[earlier]
for group in reversed(tuple(group for group in groups if group[0] <= index)[-20:])
for earlier in reversed(group)
if earlier <= index
),
)
for (index, _), ttl in zip(markers, ttls)
),
)
@dataclass(frozen=True, slots=True)
class PromptPrefix:
prefix_body: Mapping[str, JsonValue]
@ -137,68 +420,28 @@ class PromptPrefix:
def _digest(value: object) -> str:
return hashlib.sha256(
json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
json.dumps(value, default=_json_object, separators=(",", ":"), ensure_ascii=False).encode()
).hexdigest()
def _next_digest(previous: str, boundary: tuple[int, str, Mapping[str, JsonValue]]) -> str:
return _digest((previous, boundary))
def _json_object(value: object) -> dict[str, JsonValue]: # mutable-ok: JSON serialization requires a dictionary
return _JSON_OBJECT.validate_python(value)
def parse_prompt(body: Mapping[str, JsonValue]) -> PromptPrefix | None:
try:
request: Final = _Request.model_validate(body)
blocks: Final = tuple(message.blocks() for message in request.messages)
_Request.model_validate(body)
except ValidationError:
return None
markers: Final = tuple(
(message_index, block_index, block.cache_control)
for message_index, message_blocks in enumerate(blocks)
for block_index, block in enumerate(message_blocks)
if block.cache_control is not None
)
if len(markers) != 1:
plan: Final = parse_cache_plan(body)
if isinstance(plan, UnsupportedCachePlan) or len(plan.breakpoints) != 1:
return None
message_end, block_end, marker = markers[0]
normalized: Final = _JSON_OBJECT.validate_python(request.model_dump(mode="json", exclude_none=True))
context: Final = MappingProxyType({key: normalized[key] for key in ("system", "tools") if key in normalized})
boundaries: Final = tuple(
(
message_index,
request.messages[message_index].role,
_JSON_OBJECT.validate_python(
block.model_dump(mode="json", exclude=MappingProxyType({"cache_control": True}), exclude_none=True)
),
)
for message_index, message_blocks in enumerate(blocks[: message_end + 1])
for block_index, block in enumerate(message_blocks)
if message_index < message_end or block_index <= block_end
)
hashes: Final = tuple(
accumulate(boundaries, _next_digest, initial=_digest((_JSON_OBJECT.validate_python(context), marker.ttl)))
)[1:]
prefix_messages: Final = tuple(
_Message(
role=request.messages[message_index].role,
content=tuple(
block
for block_index, block in enumerate(message_blocks)
if message_index < message_end or block_index <= block_end
),
)
for message_index, message_blocks in enumerate(blocks[: message_end + 1])
)
prefix: Final = plan.breakpoints[0]
return PromptPrefix(
prefix_body=MappingProxyType(
_JSON_OBJECT.validate_python(
_Request(messages=prefix_messages, system=request.system, tools=request.tools).model_dump(
mode="json", exclude_none=True
)
)
),
fingerprint=hashes[-1],
fingerprints=tuple(reversed(hashes[-20:])),
ttl_seconds=3600 if marker.ttl == "1h" else 300,
prefix_body=prefix.prefix_body,
fingerprint=prefix.fingerprint,
fingerprints=prefix.lookback_fingerprints,
ttl_seconds=prefix.ttl_seconds,
)
@ -246,6 +489,9 @@ class _CountBody(BaseModel):
messages: Sequence[Mapping[str, JsonValue]]
tools: Sequence[Mapping[str, JsonValue]] | None = None
system: str | Sequence[Mapping[str, JsonValue]] | None = None
thinking: Mapping[str, JsonValue] | None = None
tool_choice: Mapping[str, JsonValue] | None = None
output_config: Mapping[str, JsonValue] | None = None
class _CountResult(BaseModel):
@ -262,16 +508,36 @@ def _count_objects(
return [dict(value) for value in values] # mutable-ok: serialize read-only inputs at the provider API boundary
async def count_prompt_tokens(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
native: Final = _CountBody.model_validate(body)
def _messages_url(model: str, api_key: str, api_base: str | None) -> str:
return AnthropicMessagesConfig().get_complete_url( # pyright: ignore[reportUnknownMemberType] # canonical native URL owner takes legacy JSON arguments
api_base=api_base,
api_key=api_key,
model=model,
optional_params=_JSON_OBJECT.validate_python(MappingProxyType({})),
litellm_params=_JSON_OBJECT.validate_python(MappingProxyType({})),
)
async def count_prompt_tokens(
model: str,
api_key: str,
body: Mapping[str, JsonValue],
api_base: str | None = None,
) -> int | None:
try:
native: Final = _CountBody.model_validate(body)
count_url: Final = _messages_url(model, api_key, api_base) + "/count_tokens"
result: Final = _CountResult.model_validate(
await _counter.handle_count_tokens_request(
model=model,
messages=_count_objects(native.messages),
tools=_count_objects(native.tools) if native.tools is not None else None,
system=native.system,
system=_JSON_OBJECT.validate_python(MappingProxyType({"system": native.system}))["system"],
api_key=api_key,
api_base=count_url,
optional_params=_JSON_OBJECT.validate_python(
MappingProxyType({key: body[key] for key in COUNT_TOKEN_OPTION_NAMES if key in body})
),
timeout=15.0,
)
)
@ -280,10 +546,55 @@ async def count_prompt_tokens(model: str, api_key: str, body: Mapping[str, JsonV
return result.input_tokens
async def count_cache_plan(
model: str,
api_key: str,
plan: PromptCachePlan,
token_counter: TokenCounter = count_prompt_tokens,
) -> CountedPromptCachePlan | UnsupportedCachePlan:
if any(position.block.get("type") == "thinking" for position in _positions(plan.full_body)):
if not supports_thinking_cache_preservation(model, "anthropic"):
return UnsupportedCachePlan("unsupported_thinking_cache_semantics")
total: Final = await token_counter(model, api_key, plan.full_body)
if total is None:
return UnsupportedCachePlan("token_count_unavailable")
counts: Final = tuple(
await asyncio.gather(*(token_counter(model, api_key, marker.prefix_body) for marker in plan.breakpoints))
)
if any(value is None for value in counts):
return UnsupportedCachePlan("token_count_unavailable")
known: Final = tuple(value for value in counts if value is not None)
if any(value < 0 for value in (total, *known)) or any(
first > second for first, second in zip(known, (*known[1:], total))
):
return UnsupportedCachePlan("inconsistent_prefix_token_count")
return CountedPromptCachePlan(
total,
tuple(
CountedBreakpoint(
marker.fingerprint,
marker.ttl_seconds,
count,
marker.lookback_fingerprints,
marker.content_fingerprint,
marker.lookback_content_fingerprints,
)
for marker, count in zip(plan.breakpoints, known)
),
)
@dataclass(frozen=True, slots=True)
class NativePredictionTarget:
model: str
api_key: str
api_key: str = field(repr=False)
api_base: str | None = None
def supported_baseline_recipient(target: NativePredictionTarget, wire: httpx.Request) -> bool:
return wire.headers.get("x-api-key") == target.api_key and wire.url == httpx.URL(
_messages_url(target.model, target.api_key, target.api_base)
)
@dataclass(frozen=True, slots=True)
@ -297,11 +608,26 @@ class UnsupportedPredictionTarget:
def resolve_prediction_target(params: LiteLLM_Params) -> NativePredictionTarget | UnsupportedPredictionTarget:
return _resolve_prediction_target(params, allow_configured_endpoint=False)
def resolve_baseline_prediction_target(params: LiteLLM_Params) -> NativePredictionTarget | UnsupportedPredictionTarget:
return _resolve_prediction_target(params, allow_configured_endpoint=True)
def _resolve_prediction_target(
params: LiteLLM_Params,
*,
allow_configured_endpoint: bool,
) -> NativePredictionTarget | UnsupportedPredictionTarget:
configured_options: Final = frozenset(params.model_dump(exclude_defaults=True, exclude_none=True))
if configured_options - _DEPLOYMENT_OPTIONS:
return UnsupportedPredictionTarget("unsupported_deployment_configuration")
api_base: Final = AnthropicModelInfo.get_api_base(params.api_base)
if api_base not in ("https://api.anthropic.com", "https://api.anthropic.com/v1/messages"):
if not allow_configured_endpoint and api_base not in (
"https://api.anthropic.com",
"https://api.anthropic.com/v1/messages",
):
return UnsupportedPredictionTarget("unsupported_provider_endpoint")
try:
model, provider, _, _ = litellm.get_llm_provider(
@ -314,7 +640,7 @@ def resolve_prediction_target(params: LiteLLM_Params) -> NativePredictionTarget
api_key: Final = AnthropicModelInfo.get_api_key(params.api_key)
if api_key is None or not _supported_provider_key(api_key):
return UnsupportedPredictionTarget("unsupported_provider_credentials")
return NativePredictionTarget(model=model, api_key=api_key)
return NativePredictionTarget(model=model, api_key=api_key, api_base=api_base)
def _supported_provider_key(api_key: str) -> bool:

2
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -2081,6 +2081,8 @@ class BaseLLMHTTPHandler:
e=e, litellm_params=litellm_params_dict
)
if should_retry and not hit_max_attempt:
if logging_obj.baseline_cache_context is not None:
await logging_obj.invalidate_baseline_cache_estimate("retried_request")
verbose_logger.debug(
"Anthropic /v1/messages: invalid thinking signature; "
"stripping thinking blocks and retrying (attempt %s/%s).",

10
litellm/model_prices_and_context_window_backup.json
View file

@ -13562,6 +13562,7 @@
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
@ -13600,6 +13601,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -13752,6 +13754,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -13782,6 +13785,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -13815,6 +13819,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -13853,6 +13858,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -13889,6 +13895,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -13928,6 +13935,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -14048,6 +14056,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -14090,6 +14099,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,

15
litellm/models/autorouter_session.py
View file

@ -8,6 +8,8 @@ maintains per (api_key, session_id, router_name).
from collections.abc import Mapping
from datetime import datetime
from pydantic import Field
from litellm.types.llms.base import LiteLLMPydanticObjectBase
@ -22,18 +24,25 @@ class LiteLLM_AutoRouterSession(LiteLLMPydanticObjectBase):
turns: int
spend: float
saved_spend: float
savings_estimated_turns: int = 0
savings_estimated_actual_spend: float = 0.0
savings_estimated_saved_spend: float = 0.0
savings_estimated_baseline_models: Mapping[str, int] = Field(default_factory=dict)
classifier_cost: float
tier_turns: Mapping[str, int]
baseline_models: Mapping[str, int]
@property
def baseline_model(self) -> str | None:
"""The baseline most of this session's turns were priced against, or None when no turn recorded one.
"""The baseline most covered turns were priced against, or None when none were estimated.
A router reconfigured mid-session leaves turns priced against two baselines; the row keeps both
counts, and the label is the one that priced the most money-carrying turns rather than whatever the
router is configured with now.
"""
if not self.baseline_models:
if not self.savings_estimated_baseline_models:
return None
return max(self.baseline_models, key=lambda model: (self.baseline_models[model], model))
return max(
self.savings_estimated_baseline_models,
key=lambda model: (self.savings_estimated_baseline_models[model], model),
)

5
litellm/proxy/_types.py
View file

@ -13,6 +13,7 @@ from pydantic import (
ConfigDict,
Field,
Json,
JsonValue,
PositiveInt,
field_validator,
model_validator,
@ -3833,6 +3834,7 @@ class SpendLogsRouterMetadata(TypedDict):
class SpendLogsMetadata(TypedDict):
autorouter_baseline_observation: ReadOnly[str | None]
"""
Specific metadata k,v pairs logged to spendlogs for easier cost tracking
"""
@ -3873,7 +3875,8 @@ class SpendLogsMetadata(TypedDict):
original_model_group: ReadOnly[str | None] # Model group requested before any fallbacks
cost_breakdown: CostBreakdown | None # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
compression_savings: CompressionSavingsMetadata | None
autorouter_savings: ReadOnly[float | None] # stamped by the logging payload; None = not auto-routed
autorouter_savings: ReadOnly[float | None]
autorouter_savings_estimate: ReadOnly[Mapping[str, JsonValue] | None]
litellm_gateway_injected_cache: ReadOnly[str | None]
router_metadata: ReadOnly[SpendLogsRouterMetadata | None] # None = deployment not flagged internal_router_model

67
litellm/proxy/client/cli/commands/statusline_script.py
View file

@ -32,6 +32,7 @@ import unicodedata
import urllib.error
import urllib.request
from collections.abc import Callable, Mapping
from math import isfinite
from pathlib import Path
from types import MappingProxyType
from typing import IO, Final, NamedTuple, Protocol
@ -43,6 +44,7 @@ FETCH_TIMEOUT_SECONDS: Final = 3
BAR_WIDTH: Final = 24
BAR_FULL: Final = "\u2588"
BAR_EMPTY: Final = "\u2591"
SEPARATOR: Final = " \u00b7 "
TRANSCRIPT_SCAN_LIMIT_BYTES: Final = 4 * 1024 * 1024
CLAUDE_BASE_URL_ENV_KEYS: Final = ("ANTHROPIC_BASE_URL",)
CLAUDE_API_KEY_ENV_KEYS: Final = ("ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_KEY")
@ -63,8 +65,11 @@ class Session(NamedTuple):
router_name: str
last_model: str
spend: float
baseline_spend: float
baseline_spend: float | None
baseline_model: str | None
turns: int | None = None
savings_estimated_turns: int | None = None
savings_estimated_actual_spend: float | None = None
class Credentials(NamedTuple):
@ -205,17 +210,38 @@ def _session_from_payload(payload: Mapping[str, object]) -> Session | None:
router_name: Final = printable(payload.get("router_name"))
last_model: Final = printable(payload.get("last_model"))
spend: Final = payload.get("spend")
baseline_spend: Final = payload.get("baseline_spend")
baseline_spend: Final = payload.get("savings_estimated_baseline_spend", payload.get("baseline_spend"))
turns: Final = payload.get("turns")
estimated_turns: Final = payload.get("savings_estimated_turns")
estimated_actual: Final = payload.get("savings_estimated_actual_spend")
if not router_name or not last_model:
return None
if not isinstance(spend, (int, float)) or not isinstance(baseline_spend, (int, float)):
if not isinstance(spend, (int, float)) or isinstance(spend, bool) or not isfinite(spend):
return None
if baseline_spend is not None and (
not isinstance(baseline_spend, (int, float)) or isinstance(baseline_spend, bool) or not isfinite(baseline_spend)
):
return None
return Session(
router_name=router_name,
last_model=last_model,
spend=float(spend),
baseline_spend=float(baseline_spend),
baseline_spend=float(baseline_spend) if baseline_spend is not None else None,
baseline_model=printable(payload.get("baseline_model")) or None,
turns=turns if isinstance(turns, int) and not isinstance(turns, bool) and turns >= 0 else None,
savings_estimated_turns=(
estimated_turns
if isinstance(estimated_turns, int) and not isinstance(estimated_turns, bool) and estimated_turns >= 0
else (0 if estimated_turns is not None else None)
),
savings_estimated_actual_spend=(
float(estimated_actual)
if isinstance(estimated_actual, (int, float))
and not isinstance(estimated_actual, bool)
and isfinite(estimated_actual)
and estimated_actual >= 0
else None
),
)
@ -314,15 +340,36 @@ def render(model: str, session: Session | None, config_dir: Path, use_color: boo
return f"{code}{text}{RESET}" if use_color else text
routed: Final = paint(BOLD, f"Routed to: {model}")
if session is None or session.baseline_model is None or session.baseline_spend <= 0:
if session is None:
return routed
if session.savings_estimated_turns == 0 or session.baseline_spend is None:
return f"{routed}{SEPARATOR}Savings unavailable"
if session.baseline_model is None or session.baseline_spend <= 0:
return routed
if session.savings_estimated_turns is not None and (
session.savings_estimated_actual_spend is None
or session.turns is None
or session.savings_estimated_turns > session.turns
):
return f"{routed}{SEPARATOR}Savings unavailable"
compared_spend: Final = (
session.savings_estimated_actual_spend
if session.savings_estimated_turns is not None and session.savings_estimated_actual_spend is not None
else session.spend
)
coverage: Final = (
f"{SEPARATOR}{session.savings_estimated_turns} of {session.turns} turns estimated"
if session.savings_estimated_turns is not None
else ""
)
reference: Final = baseline_label(session.baseline_model, config_dir)
pct: Final = (session.baseline_spend - session.spend) / session.baseline_spend * 100
delta: Final = paint(LITELLM_COLOR, f"{'-' if pct >= 0 else '+'}{abs(round(pct))}% vs {reference}")
peak: Final = max(session.spend, session.baseline_spend)
pct: Final = round((session.baseline_spend - compared_spend) / session.baseline_spend * 100)
sign: Final = "-" if pct > 0 else "+" if pct < 0 else ""
delta: Final = paint(LITELLM_COLOR, f"{sign}{abs(pct)}% vs {reference}")
peak: Final = max(compared_spend, session.baseline_spend)
label_width: Final = max(_display_width(session.router_name), _display_width(reference))
rows: Final = (
(session.router_name, session.spend, LITELLM_COLOR),
(session.router_name, compared_spend, LITELLM_COLOR),
(reference, session.baseline_spend, BASELINE_COLOR),
)
lines: Final = (
@ -331,7 +378,7 @@ def render(model: str, session: Session | None, config_dir: Path, use_color: boo
f"{paint(DIM, f'${amount:.2f}')}"
for label, amount, color in rows
)
return "\n".join((f"{routed} {delta}", *lines))
return "\n".join((f"{routed} {delta}{coverage}", *lines))
def color_enabled(env: Mapping[str, str]) -> bool:

8
litellm/proxy/common_request_processing.py
View file

@ -3642,6 +3642,14 @@ class ProxyBaseLLMRequestProcessing:
"async_streaming_data_generator: error closing response stream: %s",
e,
)
logging_obj: Final = request_data.get("litellm_logging_obj")
if (
not stream_completed
and isinstance(logging_obj, LiteLLMLoggingObj)
and logging_obj.baseline_cache_context is not None
and logging_obj.model_call_details.get("prompt_cache_response_complete") is not True
):
await logging_obj.invalidate_baseline_cache_estimate("incomplete_response", completed=True)
@staticmethod
async def async_streaming_data_generator(

43
litellm/proxy/db/autorouter_session_rollup.py
View file

@ -26,6 +26,7 @@ from typing import TYPE_CHECKING, Final, NamedTuple
from litellm._logging import verbose_proxy_logger
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES
from litellm.proxy.db.create_views import SupportsExecuteRaw
if TYPE_CHECKING:
from litellm.proxy._types import SpendLogsPayload
@ -75,6 +76,9 @@ SELECT
COALESCE(SUM(total_tokens), 0)::bigint AS total_tokens,
COALESCE(SUM(spend), 0)::float8 AS spend,
COALESCE(SUM(saved_spend), 0)::float8 AS saved_spend,
COALESCE(SUM(savings_estimated_turns), 0)::int AS savings_estimated_turns,
COALESCE(SUM(savings_estimated_actual_spend), 0)::float8 AS savings_estimated_actual_spend,
COALESCE(SUM(savings_estimated_saved_spend), 0)::float8 AS savings_estimated_saved_spend,
COALESCE(SUM(classifier_cost), 0)::float8 AS classifier_cost,
COALESCE(SUM(classifier_cost_recorded_turns), 0)::int AS classifier_cost_recorded_turns,
COALESCE(SUM(EXTRACT(EPOCH FROM (last_turn_at - first_turn_at))), 0)::float8 AS session_seconds
@ -104,6 +108,9 @@ class AutoRouterTurnTransaction:
cache_touched: bool
tier: str | None = None
baseline_model: str | None = None
savings_estimated_turns: int = 0
savings_estimated_actual_spend: float = 0.0
savings_estimated_saved_spend: float = 0.0
class TurnCacheFacts(NamedTuple):
@ -215,13 +222,18 @@ def build_autorouter_turn_transaction(
turn_at: Final = _turn_time_utc(str(payload.get("startTime") or ""))
if turn_at is None:
return None
from litellm.proxy.spend_tracking.savings import classifier_cost_from_decision
from litellm.proxy.spend_tracking.savings import (
classifier_cost_from_decision,
recorded_estimated_autorouter_savings,
)
usage_object_raw: Final = metadata.get("usage_object")
cache: Final = turn_cache_facts(usage_object_raw if isinstance(usage_object_raw, Mapping) else None)
tier_raw: Final = routing_decision.get("tier")
baseline_raw: Final = routing_decision.get("savings_baseline_model")
classifier_cost: Final = classifier_cost_from_decision(routing_decision)
actual_spend: Final = float(payload.get("spend") or 0.0) + (classifier_cost or 0.0)
estimated_savings: Final = recorded_estimated_autorouter_savings(metadata)
return AutoRouterTurnTransaction(
api_key=api_key,
session_id=bounded_session_id(session_id),
@ -232,13 +244,16 @@ def build_autorouter_turn_transaction(
model=model,
turn_at=turn_at,
total_tokens=int(payload.get("prompt_tokens") or 0) + int(payload.get("completion_tokens") or 0),
spend=float(payload.get("spend") or 0.0) + (classifier_cost or 0.0),
spend=actual_spend,
saved_spend=saved_spend,
classifier_cost=classifier_cost or 0.0,
covered=cache.covered,
cache_hit=cache.read_tokens > 0,
cache_ttl_seconds=cache.write_ttl_seconds,
cache_touched=cache.touched,
savings_estimated_turns=int(estimated_savings is not None),
savings_estimated_actual_spend=actual_spend if estimated_savings is not None else 0.0,
savings_estimated_saved_spend=estimated_savings if estimated_savings is not None else 0.0,
)
@ -263,6 +278,10 @@ _BASELINE: Final = f"{_p('baseline_model')}::text"
_BASELINE_DELTA: Final = (
f"(CASE WHEN {_BASELINE} IS NULL THEN '{{}}'::jsonb ELSE jsonb_build_object({_BASELINE}, 1) END)"
)
_ESTIMATED_BASELINE: Final = f"{_p('savings_estimated_turns')}::int = 1 AND {_BASELINE} IS NOT NULL"
_ESTIMATED_BASELINE_DELTA: Final = (
f"(CASE WHEN {_ESTIMATED_BASELINE} THEN jsonb_build_object({_BASELINE}, 1) ELSE '{{}}'::jsonb END)"
)
_IN_ORDER: Final = f"{_TURN_AT}::timestamp >= t.last_turn_at"
_SAME: Final = f"{_IN_ORDER} AND t.last_model = {_MODEL}"
@ -281,7 +300,8 @@ INSERT INTO "LiteLLM_AutoRouterSession" AS t (
same_model_turns, same_model_hits, first_visit_turns, first_visit_hits,
return_turns, return_hits, return_expired_misses, return_within_ttl_misses,
ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend, classifier_cost, classifier_cost_recorded_turns, tier_turns,
baseline_models
baseline_models, savings_estimated_turns, savings_estimated_actual_spend, savings_estimated_saved_spend,
savings_estimated_baseline_models
)
VALUES (
{_p("api_key")}, {_p("session_id")}, {_p("router_name")}, {_p("router_type")}, {_TURN_AT}::timestamp, {_TURN_AT}::timestamp,
@ -292,13 +312,18 @@ VALUES (
(CASE WHEN {_CACHE_TTL}::int = {CACHE_TTL_5M_SECONDS} THEN 1 ELSE 0 END),
(CASE WHEN {_CACHE_TTL}::int = {CACHE_TTL_1H_SECONDS} THEN 1 ELSE 0 END),
{_p("total_tokens")}::bigint, {_p("spend")}::float8, {_p("saved_spend")}::float8,
{_p("classifier_cost")}::float8, 1, {_TIER_DELTA}, {_BASELINE_DELTA}
{_p("classifier_cost")}::float8, 1, {_TIER_DELTA}, {_BASELINE_DELTA},
{_p("savings_estimated_turns")}::int, {_p("savings_estimated_actual_spend")}::float8,
{_p("savings_estimated_saved_spend")}::float8, {_ESTIMATED_BASELINE_DELTA}
)
ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
turns = t.turns + 1,
total_tokens = t.total_tokens + EXCLUDED.total_tokens,
spend = t.spend + EXCLUDED.spend,
saved_spend = t.saved_spend + EXCLUDED.saved_spend,
savings_estimated_turns = t.savings_estimated_turns + EXCLUDED.savings_estimated_turns,
savings_estimated_actual_spend = t.savings_estimated_actual_spend + EXCLUDED.savings_estimated_actual_spend,
savings_estimated_saved_spend = t.savings_estimated_saved_spend + EXCLUDED.savings_estimated_saved_spend,
classifier_cost = t.classifier_cost + EXCLUDED.classifier_cost,
classifier_cost_recorded_turns = t.classifier_cost_recorded_turns + 1,
covered_turns = t.covered_turns + EXCLUDED.covered_turns,
@ -331,6 +356,10 @@ ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
baseline_models = (CASE WHEN {_BASELINE} IS NOT NULL
THEN t.baseline_models || jsonb_build_object({_BASELINE}, COALESCE((t.baseline_models ->> {_BASELINE})::int, 0) + 1)
ELSE t.baseline_models END),
savings_estimated_baseline_models = (CASE WHEN {_ESTIMATED_BASELINE}
THEN t.savings_estimated_baseline_models || jsonb_build_object(
{_BASELINE}, COALESCE((t.savings_estimated_baseline_models ->> {_BASELINE})::int, 0) + 1)
ELSE t.savings_estimated_baseline_models END),
first_turn_at = LEAST(t.first_turn_at, EXCLUDED.first_turn_at),
last_turn_at = GREATEST(t.last_turn_at, EXCLUDED.last_turn_at)
"""
@ -348,6 +377,10 @@ def _upsert_params(transaction: AutoRouterTurnTransaction) -> tuple[str | float
return tuple(_as_sql_param(getattr(transaction, name)) for name in _UPSERT_PARAM_FIELDS)
async def write_autorouter_turn(db: SupportsExecuteRaw, transaction: AutoRouterTurnTransaction) -> None:
await db.execute_raw(UPSERT_AUTOROUTER_SESSION_SQL, *_upsert_params(transaction))
async def _upsert_turn_with_retry(
prisma_client: PrismaClient,
transaction: AutoRouterTurnTransaction,
@ -355,7 +388,7 @@ async def _upsert_turn_with_retry(
) -> None:
for attempt in range(n_retry_times + 1):
try:
await prisma_client.db.execute_raw(UPSERT_AUTOROUTER_SESSION_SQL, *_upsert_params(transaction))
await write_autorouter_turn(prisma_client.db, transaction)
except DB_RETRY_SAFE_ERROR_TYPES:
if attempt >= n_retry_times:
raise

640
litellm/proxy/db/baseline_accounting.py Normal file
View file

@ -0,0 +1,640 @@
from __future__ import annotations
import asyncio
import json
from collections.abc import AsyncIterator, Callable, Sequence
from datetime import datetime, timedelta
from functools import reduce
from itertools import groupby
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, cast
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator
from typing_extensions import Self
from litellm._logging import verbose_proxy_logger
from litellm.proxy.db.autorouter_session_rollup import (
AutoRouterTurnTransaction,
write_autorouter_turn,
)
from litellm.proxy.db.create_views import SupportsRawQueries
from litellm.proxy.db.daily_spend_bulk_upsert import (
DAILY_SPEND_TABLES,
DailySpendEntity,
SpendRow,
build_bulk_upsert,
merge_by_conflict_key,
)
from litellm.proxy.db.routing_prisma_wrapper import writer_wrapper
from litellm.proxy.spend_tracking.baseline_accounting import (
BaselineEstimate,
BaselineHistory,
BaselineObservation,
advance_baseline_history,
)
from litellm.proxy.spend_tracking.savings import BaselineCosts, BaselineCostSnapshot, price_baseline_comparison
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
class DailyBaselineTarget(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
entity: DailySpendEntity
entity_id: str | None
class DailyBaselineAttribution(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
date: str
api_key: str
model: str | None = None
custom_llm_provider: str | None = None
model_group: str | None = None
endpoint: str | None = None
mcp_namespaced_tool_name: str | None = None
targets: tuple[DailyBaselineTarget, ...] = ()
def adjustment(self, target: DailyBaselineTarget, savings_delta: float, request_id: str) -> SpendRow:
table: Final = DAILY_SPEND_TABLES[target.entity]
return MappingProxyType(
{
"date": self.date,
"api_key": self.api_key,
"model": self.model,
"custom_llm_provider": self.custom_llm_provider,
"model_group": self.model_group,
"endpoint": self.endpoint,
"mcp_namespaced_tool_name": self.mcp_namespaced_tool_name,
table.entity_id_column: target.entity_id,
"request_id": request_id,
"autorouter_savings_spend": savings_delta,
}
)
class BaselineAccountingRecord(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
scope: str = Field(pattern=r"^autorouter-baseline:v3:[a-f0-9]{64}$")
api_key: str = Field(min_length=1)
session_id: str = Field(min_length=1, max_length=256)
router_name: str = Field(min_length=1)
baseline_model: str = Field(min_length=1)
observation: BaselineObservation
pricing: BaselineCostSnapshot
turn: AutoRouterTurnTransaction | None
daily: DailyBaselineAttribution | None
@model_validator(mode="after")
def consistent_turn(self) -> Self:
turn: Final = self.turn
if turn is not None and (
(turn.api_key, turn.session_id, turn.router_name, turn.baseline_model)
!= (self.api_key, self.session_id, self.router_name, self.baseline_model)
or turn.spend != self.pricing.actual_spend + self.pricing.classifier_cost
or any(
(
turn.saved_spend,
turn.savings_estimated_turns,
turn.savings_estimated_actual_spend,
turn.savings_estimated_saved_spend,
)
)
):
raise ValueError("Baseline observation must own an unestimated turn with matching scope and actual cost")
return self
class BaselinePublication(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
version: Literal[3] = 3
comparison_id: str
comparison_started_at: float
status: Literal["estimated", "unknown"]
reason: str
provenance: Literal["observed_identical", "modeled"] | None = None
actual_spend: float | None = None
baseline_spend: float | None = None
input_tokens: int | None = None
cache_read_input_tokens: int | None = None
cache_creation_5m_input_tokens: int | None = None
cache_creation_1h_input_tokens: int | None = None
@property
def costs(self) -> BaselineCosts | None:
if self.status != "estimated" or self.actual_spend is None or self.baseline_spend is None:
return None
return BaselineCosts(self.actual_spend, self.baseline_spend)
def baseline_publication(
record: BaselineAccountingRecord, estimate: BaselineEstimate, first_at: float
) -> BaselinePublication:
costs: Final = price_baseline_comparison(record.pricing, estimate.usage, estimate.provenance)
details: Final = estimate.usage.prompt_tokens_details if estimate.usage is not None else None
writes: Final = details.cache_creation_token_details if details is not None else None
return BaselinePublication(
comparison_id=record.scope,
comparison_started_at=first_at,
status="estimated" if costs is not None else "unknown",
reason=estimate.reason if costs is not None or estimate.usage is None else "pricing_unavailable",
provenance=estimate.provenance if costs is not None else None,
actual_spend=costs.actual if costs is not None else None,
baseline_spend=costs.baseline if costs is not None else None,
input_tokens=details.text_tokens if details is not None else None,
cache_read_input_tokens=details.cached_tokens if details is not None else None,
cache_creation_5m_input_tokens=writes.ephemeral_5m_input_tokens if writes is not None else None,
cache_creation_1h_input_tokens=writes.ephemeral_1h_input_tokens if writes is not None else None,
)
class _Comparison(BaseModel):
revision: int
published_revision: int
initial_equivalent: bool
retired: bool
history: str | None
class _StoredRecord(BaseModel):
data: str
publication: str | None
conflicted: bool
started_at: float
class _Change(BaseModel):
request_id: str
publication: BaselinePublication
api_key: str
session_id: str
router_name: str
baseline_model: str
covered_delta: int
actual_delta: float
savings_delta: float
daily: DailyBaselineAttribution | None
class _TransactionManager(Protocol):
async def __aenter__(self) -> SupportsRawQueries: ...
async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ...
class _TransactionalDatabase(Protocol):
def tx(self, *, timeout: timedelta) -> _TransactionManager: ...
_COMPARISONS: Final = TypeAdapter(tuple[_Comparison, ...])
_RECORDS: Final = TypeAdapter(tuple[_StoredRecord, ...])
_HISTORY: Final = TypeAdapter(BaselineHistory)
_PAGE_TIMESTAMPS: Final = 128
_TRANSACTION_TIMEOUT: Final = timedelta(seconds=10)
_CREATE_COMPARISON: Final = """
INSERT INTO "LiteLLM_AutoRouterBaselineComparison"
(scope, api_key, session_id, router_name, initial_equivalent)
VALUES ($1, $2, $3, $4, NOT EXISTS (
SELECT 1 FROM "LiteLLM_AutoRouterSession"
WHERE api_key = $2 AND session_id = $3 AND router_name = $4
)) ON CONFLICT (scope) DO NOTHING
"""
_LOCK_COMPARISON: Final = """
SELECT revision, published_revision, initial_equivalent, retired, history
FROM "LiteLLM_AutoRouterBaselineComparison" WHERE scope = $1 FOR UPDATE
"""
_INSERT_RECORD: Final = """
INSERT INTO "LiteLLM_AutoRouterBaselineObservation"
(request_id, scope, started_at, revision, data)
VALUES ($1, $2, $3::float8, $4::bigint, $5)
ON CONFLICT (request_id) DO NOTHING
"""
_MARK_CONFLICT: Final = """
UPDATE "LiteLLM_AutoRouterBaselineObservation"
SET conflicted = TRUE, revision = $4::bigint
WHERE request_id = $1 AND scope = $2 AND data <> $3 AND NOT conflicted
"""
_READ_PAGE: Final = """
WITH times AS (
SELECT DISTINCT started_at FROM "LiteLLM_AutoRouterBaselineObservation"
WHERE scope = $1 AND revision > $2::bigint
AND ($3::float8 IS NULL OR started_at > $3::float8)
AND ($5::float8 IS NULL OR (
started_at >= $5::float8 AND publication::jsonb->>'status' = 'estimated'
))
ORDER BY started_at LIMIT $4::int
)
SELECT data, publication, conflicted, started_at
FROM "LiteLLM_AutoRouterBaselineObservation"
WHERE scope = $1 AND revision > $2::bigint
AND started_at IN (SELECT started_at FROM times)
AND ($5::float8 IS NULL OR publication::jsonb->>'status' = 'estimated')
ORDER BY started_at, request_id
"""
_UPDATE_LOGS: Final = """
WITH changes AS (
SELECT request_id, publication::jsonb AS publication
FROM jsonb_to_recordset($1::jsonb) AS x(request_id text, publication jsonb)
)
UPDATE "LiteLLM_SpendLogs" AS logs
SET metadata = (COALESCE(logs.metadata::jsonb, '{}'::jsonb) - 'autorouter_baseline_observation') || jsonb_build_object(
'autorouter_savings_estimate', changes.publication,
'autorouter_savings', CASE WHEN changes.publication->>'status' = 'estimated' THEN
(changes.publication->>'baseline_spend')::float8 - (changes.publication->>'actual_spend')::float8
ELSE NULL END
)
FROM changes WHERE logs.request_id = changes.request_id
"""
_UPDATE_PUBLICATIONS: Final = """
UPDATE "LiteLLM_AutoRouterBaselineObservation" AS observations
SET publication = x.publication::text
FROM jsonb_to_recordset($1::jsonb) AS x(request_id text, publication jsonb)
WHERE observations.request_id = x.request_id
"""
_UPDATE_SESSIONS: Final = """
WITH changes AS (
SELECT * FROM jsonb_to_recordset($1::jsonb) AS x(
api_key text, session_id text, router_name text, baseline_model text,
covered_delta int, actual_delta float8, savings_delta float8
)
), totals AS (
SELECT api_key, session_id, router_name, SUM(covered_delta)::int AS covered_delta,
SUM(actual_delta) AS actual_delta, SUM(savings_delta) AS savings_delta
FROM changes GROUP BY api_key, session_id, router_name
), models AS (
SELECT api_key, session_id, router_name, jsonb_object_agg(baseline_model, delta) AS deltas
FROM (
SELECT api_key, session_id, router_name, baseline_model, SUM(covered_delta)::int AS delta
FROM changes GROUP BY api_key, session_id, router_name, baseline_model
) grouped GROUP BY api_key, session_id, router_name
)
UPDATE "LiteLLM_AutoRouterSession" AS session
SET saved_spend = session.saved_spend + totals.savings_delta,
savings_estimated_turns = session.savings_estimated_turns + totals.covered_delta,
savings_estimated_actual_spend = session.savings_estimated_actual_spend + totals.actual_delta,
savings_estimated_saved_spend = session.savings_estimated_saved_spend + totals.savings_delta,
savings_estimated_baseline_models = (
SELECT COALESCE(jsonb_object_agg(key, value), '{}'::jsonb) FROM (
SELECT key, SUM(value::int)::int AS value FROM (
SELECT * FROM jsonb_each_text(session.savings_estimated_baseline_models)
UNION ALL SELECT * FROM jsonb_each_text(models.deltas)
) combined GROUP BY key HAVING SUM(value::int) > 0
) counts
)
FROM totals JOIN models USING (api_key, session_id, router_name)
WHERE session.api_key = totals.api_key AND session.session_id = totals.session_id
AND session.router_name = totals.router_name
"""
def _primary_transaction(client: PrismaClient) -> _TransactionManager:
primary: Final = cast(_TransactionalDatabase, writer_wrapper(client.db))
return primary.tx(timeout=_TRANSACTION_TIMEOUT)
def _serialized(model: BaseModel) -> str:
return json.dumps(model.model_dump(mode="json"), sort_keys=True, separators=(",", ":"))
def _change(record: BaselineAccountingRecord, old: BaselinePublication | None, new: BaselinePublication) -> _Change:
previous: Final = old.costs if old is not None else None
current: Final = new.costs
return _Change(
request_id=record.observation.request_id,
publication=new,
api_key=record.api_key,
session_id=record.session_id,
router_name=record.router_name,
baseline_model=record.baseline_model,
covered_delta=int(current is not None) - int(previous is not None),
actual_delta=(current.actual if current is not None else 0.0)
- (previous.actual if previous is not None else 0.0),
savings_delta=(current.savings if current is not None else 0.0)
- (previous.savings if previous is not None else 0.0),
daily=record.daily,
)
def _project_group(
previous: tuple[BaselineHistory, tuple[_Change, ...]], stored: Sequence[_StoredRecord]
) -> tuple[BaselineHistory, tuple[_Change, ...]]:
history, prior_changes = previous
records: Final = tuple(BaselineAccountingRecord.model_validate_json(item.data) for item in stored)
observations: Final = tuple(
record.observation.model_copy(
update=MappingProxyType(
{"outcome": "uncertain", "baseline_equivalent": False, "reason": "conflicting_observation"}
)
)
if row.conflicted
else record.observation
for record, row in zip(records, stored)
)
advanced, estimates = advance_baseline_history(history, observations)
publications: Final = tuple(
baseline_publication(
record, estimate, advanced.first_at if advanced.first_at is not None else observations[0].started_at
)
for record, estimate in zip(records, estimates)
)
changes: Final = tuple(
_change(record, old, publication)
for record, row, publication in zip(records, stored, publications)
for old in (BaselinePublication.model_validate_json(row.publication) if row.publication else None,)
if publication != old
)
return advanced, (*prior_changes, *changes)
async def _publish(db: SupportsRawQueries, changes: Sequence[_Change]) -> None:
if not changes:
return
serialized: Final = json.dumps(tuple(change.model_dump(mode="json") for change in changes), separators=(",", ":"))
await db.execute_raw(_UPDATE_LOGS, serialized)
await db.execute_raw(_UPDATE_SESSIONS, serialized)
for entity, table in DAILY_SPEND_TABLES.items():
if adjustments := tuple(
change.daily.adjustment(target, change.savings_delta, change.request_id)
for change in changes
if change.daily is not None and change.savings_delta != 0
for target in change.daily.targets
if target.entity == entity
):
statement, values = build_bulk_upsert(table, merge_by_conflict_key(table, adjustments))
await db.execute_raw(statement, *values)
await db.execute_raw(_UPDATE_PUBLICATIONS, serialized)
class BaselineAccountingStore:
def __init__(self, transaction: Callable[[], _TransactionManager]) -> None:
self.transaction: Final = transaction
@classmethod
def for_client(cls, client: PrismaClient) -> BaselineAccountingStore:
def transaction() -> _TransactionManager:
return _primary_transaction(client)
return cls(transaction)
async def append(
self, record: BaselineAccountingRecord
) -> Literal["recorded", "retired", "conflict", "unavailable"]:
try:
async with self.transaction() as db:
await db.execute_raw("SET LOCAL statement_timeout = 5000")
await db.execute_raw("SET LOCAL lock_timeout = 1000")
await db.execute_raw(
_CREATE_COMPARISON, record.scope, record.api_key, record.session_id, record.router_name
)
rows: Final = _COMPARISONS.validate_python(tuple(await db.query_raw(_LOCK_COMPARISON, record.scope)))
if not rows:
return "unavailable"
revision: Final = rows[0].revision + 1
data: Final = _serialized(record)
inserted: Final = await db.execute_raw(
_INSERT_RECORD,
record.observation.request_id,
record.scope,
record.observation.started_at,
revision,
data,
)
if inserted and record.turn is not None:
await write_autorouter_turn(db, record.turn)
conflicted: Final = (
0
if inserted
else await db.execute_raw(
_MARK_CONFLICT, record.observation.request_id, record.scope, data, revision
)
)
canonical: Final = (
_RECORDS.validate_python(
tuple(
await db.query_raw(
'SELECT data, publication, conflicted, started_at FROM "LiteLLM_AutoRouterBaselineObservation" '
"WHERE request_id=$1 AND scope=$2",
record.observation.request_id,
record.scope,
)
)
)
if not inserted
else ()
)
if not inserted and not canonical:
return "conflict"
if rows[0].retired:
await _publish(
db,
(
_change(
BaselineAccountingRecord.model_validate_json(canonical[0].data)
if canonical
else record,
BaselinePublication.model_validate_json(canonical[0].publication)
if canonical and canonical[0].publication is not None
else None,
BaselinePublication(
comparison_id=record.scope,
comparison_started_at=canonical[0].started_at
if canonical
else record.observation.started_at,
status="unknown",
reason="comparison_retired",
),
),
),
)
return "retired"
if inserted or conflicted:
await self._withdraw(
db, record.scope, canonical[0].started_at if canonical else record.observation.started_at
)
await db.execute_raw(
'UPDATE "LiteLLM_AutoRouterBaselineComparison" SET revision = $2::bigint, '
"updated_at = CURRENT_TIMESTAMP, attempted_at = NULL WHERE scope = $1",
record.scope,
revision,
)
return "recorded"
except Exception: # noqa: BLE001 # accounting failure must not change inference or actual billing
verbose_proxy_logger.warning("Auto-router baseline observation could not be persisted")
return "unavailable"
async def _pages(
self, db: SupportsRawQueries, scope: str, after_revision: int, withdraw_from: float | None = None
) -> AsyncIterator[tuple[_StoredRecord, ...]]:
cursor: float | None = None
while page := _RECORDS.validate_python(
tuple(await db.query_raw(_READ_PAGE, scope, after_revision, cursor, _PAGE_TIMESTAMPS, withdraw_from))
):
yield page
cursor = page[-1].started_at # rebind-ok: keyset pagination advances after each complete timestamp group
async def _withdraw(self, db: SupportsRawQueries, scope: str, started_at: float) -> None:
async for page in self._pages(db, scope, 0, withdraw_from=started_at):
await _publish(
db,
tuple(
_change(
BaselineAccountingRecord.model_validate_json(row.data),
previous,
BaselinePublication(
comparison_id=scope,
comparison_started_at=min(previous.comparison_started_at, started_at),
status="unknown",
reason="pending_projection",
),
)
for row in page
if row.publication is not None
for previous in (BaselinePublication.model_validate_json(row.publication),)
),
)
async def retire_before(self, cutoff: datetime, batch_size: int, timeout_ms: int) -> None:
async with self.transaction() as db:
await db.execute_raw(f"SET LOCAL statement_timeout = {max(1, timeout_ms)}")
await db.execute_raw(f"SET LOCAL lock_timeout = {max(1, timeout_ms)}")
await db.execute_raw(
'WITH expired AS (SELECT scope FROM "LiteLLM_AutoRouterBaselineComparison" '
"WHERE NOT retired AND updated_at < $1::timestamptz ORDER BY updated_at "
"LIMIT $2::int FOR UPDATE SKIP LOCKED) "
'UPDATE "LiteLLM_AutoRouterBaselineComparison" AS comparison '
"SET retired=TRUE, history=NULL FROM expired WHERE comparison.scope=expired.scope",
cutoff,
batch_size,
)
await db.execute_raw(
'DELETE FROM "LiteLLM_AutoRouterBaselineObservation" WHERE request_id IN ('
'SELECT event.request_id FROM "LiteLLM_AutoRouterBaselineObservation" AS event '
'JOIN "LiteLLM_AutoRouterBaselineComparison" AS comparison USING (scope) '
"WHERE comparison.retired AND comparison.updated_at < $1::timestamptz "
"LIMIT $2::int)",
cutoff,
batch_size,
)
async def project(self, scope: str) -> Literal["published", "unchanged", "unavailable"]:
try:
async with self.transaction() as db:
await db.execute_raw("SET LOCAL statement_timeout = 5000")
await db.execute_raw("SET LOCAL lock_timeout = 1000")
rows: Final = _COMPARISONS.validate_python(tuple(await db.query_raw(_LOCK_COMPARISON, scope)))
if not rows or rows[0].retired or rows[0].revision == rows[0].published_revision:
return "unchanged"
missing_log: Final = await db.query_raw(
'SELECT 1 FROM "LiteLLM_AutoRouterBaselineObservation" AS observation '
'WHERE scope=$1 AND publication IS NULL AND NOT EXISTS (SELECT 1 FROM "LiteLLM_SpendLogs" AS log '
"WHERE log.request_id=observation.request_id) LIMIT 1",
scope,
)
if missing_log:
return "unavailable"
state: Final = rows[0]
checkpoint: Final = (
_HISTORY.validate_json(state.history)
if state.history is not None
else BaselineHistory(equivalent=state.initial_equivalent)
)
changed: Final = await db.query_raw(
'SELECT 1 FROM "LiteLLM_AutoRouterBaselineObservation" '
"WHERE scope = $1 AND revision > $2::bigint AND started_at <= $3::float8 LIMIT 1",
scope,
state.published_revision,
checkpoint.last_at,
)
history = BaselineHistory(equivalent=state.initial_equivalent) if changed else checkpoint
async for page in self._pages(db, scope, 0 if changed else state.published_revision):
history, updates = reduce(
_project_group,
(tuple(group) for _, group in groupby(page, key=lambda item: item.started_at)),
(history, ()),
)
await _publish(db, updates)
await db.execute_raw(
'UPDATE "LiteLLM_AutoRouterBaselineComparison" '
"SET published_revision = revision, history = $2 WHERE scope = $1",
scope,
_HISTORY.dump_json(history).decode(),
)
return "published"
except Exception: # noqa: BLE001 # rollback leaves the durable revision dirty for a later flush
verbose_proxy_logger.warning("Auto-router baseline projection remains pending")
return "unavailable"
class _Scope(BaseModel):
scope: str
_SCOPES: Final = TypeAdapter(tuple[_Scope, ...])
_CLAIM_DIRTY: Final = """
WITH candidates AS (
SELECT scope FROM "LiteLLM_AutoRouterBaselineComparison"
WHERE NOT retired AND revision <> published_revision
AND (attempted_at IS NULL OR attempted_at < CURRENT_TIMESTAMP - INTERVAL '30 seconds')
ORDER BY attempted_at NULLS FIRST, updated_at, scope LIMIT 32 FOR UPDATE SKIP LOCKED
)
UPDATE "LiteLLM_AutoRouterBaselineComparison" AS comparison
SET attempted_at = CURRENT_TIMESTAMP FROM candidates
WHERE comparison.scope = candidates.scope RETURNING comparison.scope
"""
async def _flush_records(
store: BaselineAccountingStore, records: Sequence[BaselineAccountingRecord]
) -> tuple[BaselineAccountingRecord, ...]:
slots: Final = asyncio.Semaphore(4)
async def append(record: BaselineAccountingRecord) -> bool:
async with slots:
return await store.append(record) == "unavailable"
failed: Final = await asyncio.gather(*(append(record) for record in records))
return tuple(record for record, retry in zip(records, failed) if retry)
async def flush_baseline_accounting(client: PrismaClient) -> None:
from litellm.proxy.utils import request_spend_log_flush
store: Final = BaselineAccountingStore.for_client(client)
async with client.baseline_accounting_lock:
batch: Final = tuple(client.baseline_accounting_transactions[:32])
client.baseline_accounting_transactions = client.baseline_accounting_transactions[
32:
] # rebind-ok: drain under lock
more_queued: Final = bool(client.baseline_accounting_transactions)
try:
remaining: Final = await asyncio.wait_for(_flush_records(store, batch), timeout=5)
except (Exception, asyncio.CancelledError) as error: # noqa: BLE001 # unknown acknowledgements can be replayed safely
async with client.baseline_accounting_lock:
client.baseline_accounting_transactions.extend(batch)
if isinstance(error, asyncio.CancelledError):
raise
return
async with client.baseline_accounting_lock:
client.baseline_accounting_transactions.extend(remaining)
if more_queued and len(remaining) < len(batch):
request_spend_log_flush(client)
try:
async with store.transaction() as db:
await db.execute_raw("SET LOCAL statement_timeout = 1000")
scopes: Final = _SCOPES.validate_python(tuple(await db.query_raw(_CLAIM_DIRTY)))
slots: Final = asyncio.Semaphore(4)
async def project(item: _Scope) -> str:
async with slots:
return await store.project(item.scope)
outcomes: Final = await asyncio.wait_for(asyncio.gather(*(project(item) for item in scopes)), timeout=5)
if len(scopes) == 32 and "published" in outcomes:
request_spend_log_flush(client)
except Exception: # noqa: BLE001 # durable dirty comparisons remain eligible after the retry interval
verbose_proxy_logger.warning("Auto-router baseline projection will retry on a later spend flush")

32
litellm/proxy/db/daily_spend_bulk_upsert.py
View file

@ -14,6 +14,8 @@ from itertools import groupby
from types import MappingProxyType
from typing import Final, Literal
from pydantic import TypeAdapter
DailySpendEntity = Literal["user", "team", "org", "tag", "end_user", "agent"]
SqlValue = str | int | float | None
@ -43,6 +45,36 @@ DAILY_SPEND_TABLES: Final[Mapping[DailySpendEntity, DailySpendTable]] = MappingP
}
)
_ENTITY_INPUT_KEYS: Final[Mapping[DailySpendEntity, str]] = MappingProxyType(
{
"user": "user",
"team": "team_id",
"org": "organization_id",
"end_user": "end_user",
"agent": "agent_id",
"tag": "request_tags",
}
)
_TAGS: Final = TypeAdapter(tuple[str, ...])
def daily_spend_entity_ids(payload: Mapping[str, object], entity: DailySpendEntity) -> tuple[str | None, ...]:
key: Final = _ENTITY_INPUT_KEYS[entity]
if key not in payload:
return ()
value: Final = payload[key]
if entity == "tag":
if value is None:
return ()
tags: Final = _TAGS.validate_json(value) if isinstance(value, str) else _TAGS.validate_python(value)
return tuple(dict.fromkeys(tags))
if value is None:
return (None,) if entity == "user" else ()
if not isinstance(value, str) or (entity == "end_user" and not value):
return ()
return (value,)
# The unique constraint's columns after the entity id, in constraint order. A NULL can
# never match itself in a unique index, so every one of these is normalized to '': the
# conflict target has to be NULL-free or the row is re-inserted on every single flush.

147
litellm/proxy/db/db_spend_update_writer.py
View file

@ -18,6 +18,8 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, overload
from urllib.parse import quote, unquote
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching import RedisCache
@ -47,6 +49,7 @@ from litellm.proxy._types import (
from litellm.proxy.db.daily_spend_bulk_upsert import (
DAILY_SPEND_TABLES,
build_bulk_upsert,
daily_spend_entity_ids,
merge_by_conflict_key,
)
from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import (
@ -77,6 +80,8 @@ from litellm.repositories.prisma_protocols import BatchTable
from litellm.types.utils import CallTypes
if TYPE_CHECKING:
from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
from litellm.proxy.db.baseline_accounting import DailyBaselineAttribution
from litellm.proxy.utils import PrismaClient, ProxyLogging
else:
PrismaClient = Any
@ -84,6 +89,7 @@ else:
RESPONSES_SESSION_CALL_TYPES: Final = frozenset({CallTypes.responses.value, CallTypes.aresponses.value})
_SPEND_METADATA_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _org_member_transaction_key(org_id: str, user_id: str) -> str:
@ -502,25 +508,31 @@ class DBSpendUpdateWriter:
metadata_raw: Final = payload.get("metadata")
if not metadata_raw:
return
metadata: Final = json.loads(metadata_raw)
if not isinstance(metadata, dict) or not metadata.get("routing_decision"):
metadata: Final = _SPEND_METADATA_ADAPTER.validate_json(metadata_raw)
routing_decision: Final = metadata.get("routing_decision")
if not isinstance(routing_decision, Mapping) or not routing_decision:
return
from litellm.proxy.db.autorouter_session_rollup import (
build_autorouter_turn_transaction,
)
usage_object_raw: Final = metadata.get("usage_object")
cost_breakdown: Final = metadata.get("cost_breakdown")
savings_estimate: Final = metadata.get("autorouter_savings_estimate")
savings_spend: Final = compute_savings_spend(
model=payload.get("model"),
custom_llm_provider=payload.get("custom_llm_provider"),
compression_saved_tokens=0,
gateway_injected_cache=marks_gateway_injection(metadata, payload.get("model_id")),
routing_decision=metadata.get("routing_decision"),
routing_decision=routing_decision,
usage_object=usage_object_raw if isinstance(usage_object_raw, dict) else None,
model_id=payload.get("model_id"),
llm_router=get_llm_router,
cost_breakdown=metadata.get("cost_breakdown"),
cost_breakdown=cost_breakdown if isinstance(cost_breakdown, Mapping) else None,
recorded_autorouter_savings=metadata.get("autorouter_savings"),
recorded_autorouter_savings_estimate=(
savings_estimate if isinstance(savings_estimate, Mapping) else None
),
billed_at=payload.get("endTime"),
)
transaction: Final = build_autorouter_turn_transaction(
@ -528,6 +540,11 @@ class DBSpendUpdateWriter:
metadata=metadata,
saved_spend=savings_spend.autorouter,
)
try:
if await self._enqueue_baseline_accounting(payload, metadata, transaction, prisma_client):
return
except Exception: # noqa: BLE001 # optional baseline capture must preserve the original actual-spend rollup
verbose_proxy_logger.warning("Auto-router baseline observation was unavailable; actual turn retained")
if transaction is None:
return
async with prisma_client._autorouter_turn_transactions_lock:
@ -535,6 +552,95 @@ class DBSpendUpdateWriter:
except Exception as e: # noqa: BLE001 # a metrics enqueue must never fail the spend write
verbose_proxy_logger.debug("_enqueue_autorouter_turn_transaction error (non-blocking): %s", e)
async def _enqueue_baseline_accounting(
self,
payload: SpendLogsPayload,
metadata: Mapping[str, object],
turn: "AutoRouterTurnTransaction | None",
prisma_client: "PrismaClient",
) -> bool:
from litellm.proxy.db.baseline_accounting import (
BaselineAccountingRecord,
)
from litellm.proxy.hooks.autorouter_baseline_cache import CapturedBaselineObservation
from litellm.proxy.spend_tracking.savings import baseline_cost_snapshot
serialized: Final = metadata.get("autorouter_baseline_observation")
if not isinstance(serialized, str):
return False
captured: Final = CapturedBaselineObservation.model_validate_json(serialized)
if captured.api_key != payload["api_key"] or captured.session_id != payload["session_id"]:
return False
decision: Final = _SPEND_METADATA_ADAPTER.validate_python(
metadata.get("routing_decision") or MappingProxyType({})
)
breakdown: Final = _SPEND_METADATA_ADAPTER.validate_python(
metadata.get("cost_breakdown") or MappingProxyType({})
)
daily: Final = await self._baseline_daily_attribution(payload, prisma_client)
record: Final = BaselineAccountingRecord(
scope=captured.scope,
api_key=captured.api_key,
session_id=captured.session_id,
router_name=captured.router_name,
baseline_model=captured.baseline_model,
observation=captured.observation.model_copy(update=MappingProxyType({"request_id": payload["request_id"]})),
pricing=baseline_cost_snapshot(captured.model, captured.prices, payload["spend"], breakdown, decision),
turn=turn,
daily=daily,
)
async with prisma_client.baseline_accounting_lock:
if len(prisma_client.baseline_accounting_transactions) >= 10000:
verbose_proxy_logger.warning("Auto-router baseline observation queue is full")
return False
prisma_client.baseline_accounting_transactions.append(record)
from litellm.proxy.utils import request_spend_log_flush
request_spend_log_flush(prisma_client)
return True
async def _baseline_daily_attribution(
self,
payload: SpendLogsPayload,
prisma_client: "PrismaClient",
) -> "DailyBaselineAttribution | None":
from litellm.proxy.db.baseline_accounting import DailyBaselineAttribution, DailyBaselineTarget
normalized: Final = cast(SpendLogsPayload, MappingProxyType({**payload, "end_user_id": payload["end_user"]}))
bases: Final = tuple(
zip(
DAILY_SPEND_TABLES,
await asyncio.gather(
*(
self._common_add_spend_log_transaction_to_daily_transaction( # pyright: ignore[reportUnknownMemberType] # legacy payload union; this caller supplies a validated spend payload
normalized,
prisma_client,
"request_tags" if entity == "tag" else entity,
)
for entity in DAILY_SPEND_TABLES
)
),
)
)
base: Final = next((base for _, base in bases if base is not None), None)
if base is None:
return None
return DailyBaselineAttribution(
date=base["date"],
api_key=base["api_key"],
model=base.get("model"),
custom_llm_provider=base.get("custom_llm_provider"),
model_group=base.get("model_group"),
endpoint=base.get("endpoint"),
mcp_namespaced_tool_name=base.get("mcp_namespaced_tool_name"),
targets=tuple(
DailyBaselineTarget(entity=entity, entity_id=identity)
for entity, values in bases
if values is not None
for identity in daily_spend_entity_ids(payload, entity)
),
)
def _enqueue_tool_registry_upsert(
self,
kwargs: dict | None,
@ -2166,21 +2272,13 @@ class DBSpendUpdateWriter:
prisma_client: PrismaClient,
type: Literal["user", "team", "org", "request_tags", "end_user", "agent"] = "user",
) -> BaseDailySpendTransaction | None:
common_expected_keys: Final = ["startTime", "api_key"]
if type == "user":
expected_keys = ["user", *common_expected_keys]
elif type == "team":
expected_keys = ["team_id", *common_expected_keys]
elif type == "org":
expected_keys = ["organization_id", *common_expected_keys]
elif type == "request_tags":
expected_keys = ["request_tags", *common_expected_keys]
elif type == "end_user":
expected_keys = ["end_user_id", *common_expected_keys]
elif type == "agent":
expected_keys = ["agent_id", *common_expected_keys]
else:
raise ValueError(f"Invalid type: {type}")
entity: Final = "tag" if type == "request_tags" else type
identity_payload: Final = (
MappingProxyType({**payload, "end_user": payload.get("end_user_id")}) if type == "end_user" else payload
)
if not daily_spend_entity_ids(identity_payload, entity):
return None
expected_keys: Final = ("startTime", "api_key")
if not all(key in payload for key in expected_keys):
verbose_proxy_logger.debug(
"Missing expected keys: %s, in payload, skipping from daily_user_spend_transactions", expected_keys
@ -2243,6 +2341,7 @@ class DBSpendUpdateWriter:
usage_object=usage_obj,
cost_breakdown=_metadata.get("cost_breakdown"),
recorded_autorouter_savings=_metadata.get("autorouter_savings"),
recorded_autorouter_savings_estimate=_metadata.get("autorouter_savings_estimate"),
billed_at=payload.get("endTime"),
)
timed_duration_ms: Final = _timed_request_duration_ms(payload, request_status, is_internal_call)
@ -2441,14 +2540,10 @@ class DBSpendUpdateWriter:
verbose_proxy_logger.debug("request_tags is None for request. Skipping incrementing tag spend.")
return
request_tags: Sequence[str] = []
if isinstance(payload["request_tags"], str):
request_tags = json.loads(payload["request_tags"])
elif isinstance(payload["request_tags"], list):
request_tags = payload["request_tags"]
else:
raise ValueError(f"Invalid request_tags: {payload['request_tags']}")
request_tags: Final = daily_spend_entity_ids(payload, "tag")
for tag in request_tags:
if tag is None:
continue
endpoint_str = base_daily_transaction.get("endpoint") or ""
daily_transaction_key = f"{tag}_{base_daily_transaction['date']}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}_{endpoint_str}"
daily_transaction = DailyTagSpendTransaction(

11
litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py
View file

@ -549,6 +549,17 @@ class SpendLogCleanup:
Prune auto-router session rollup rows, which carry their own retention horizon.
"""
session_cutoff: Final = datetime.now(timezone.utc) - timedelta(seconds=float(retention_seconds))
from litellm.proxy.db.baseline_accounting import BaselineAccountingStore
if remaining_ms := self._remaining_timeout_ms(deadline)():
try:
await BaselineAccountingStore.for_client(prisma_client).retire_before(
session_cutoff,
self.batch_size,
remaining_ms,
)
except Exception: # noqa: BLE001 # retained observations are retried by the next cleanup job
verbose_proxy_logger.warning("Auto-router baseline retention remains pending")
sessions_result: Final = await self._delete_old_autorouter_session_rows(prisma_client, session_cutoff, deadline)
verbose_proxy_logger.info("Deleted %s expired auto-router session rollup rows", sessions_result.rows_deleted)
return (sessions_result,)

2
litellm/proxy/hooks/__init__.py
View file

@ -2,6 +2,7 @@ import os
from typing import Final, Literal
from . import *
from .autorouter_baseline_cache import AutoRouterBaselineCache
from .cache_control_check import _PROXY_CacheControlCheck
from .litellm_skills import SkillsInjectionHook
from .max_budget_limiter import _PROXY_MaxBudgetLimiter
@ -27,6 +28,7 @@ PROXY_HOOKS: Final = {
"max_budget_per_session_limiter": _PROXY_MaxBudgetPerSessionHandler,
"sensitive_data_routing": _PROXY_SensitiveDataRoutingHandler,
"prompt_cache_prediction": PromptCacheObserver,
"autorouter_baseline_cache": AutoRouterBaselineCache,
}
## FEATURE FLAG HOOKS ##

344
litellm/proxy/hooks/autorouter_baseline_cache.py Normal file
View file

@ -0,0 +1,344 @@
from __future__ import annotations
import asyncio
import hashlib
import json
import time
from collections.abc import Callable, Mapping
from dataclasses import dataclass, replace
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter
from litellm._logging import verbose_proxy_logger
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
get_litellm_metadata_from_kwargs, # pyright: ignore[reportUnknownVariableType] # legacy metadata boundary validated below
)
from litellm.llms.anthropic.prompt_cache_prediction import (
CountedPromptCachePlan,
NativePredictionTarget,
TokenCounter,
UnsupportedCachePlan,
UnsupportedPredictionTarget,
count_cache_plan,
count_prompt_tokens,
parse_cache_plan,
resolve_baseline_prediction_target,
supported_baseline_recipient,
supported_prediction_headers,
)
from litellm.proxy.spend_tracking.baseline_accounting import BaselineObservation
from litellm.proxy.spend_tracking.savings import (
_effective_model_info, # pyright: ignore[reportPrivateUsage] # existing deployment-price owner
_proxy_llm_router, # pyright: ignore[reportPrivateUsage] # existing optional proxy-router owner
)
from litellm.types.router import BaselineRouteStamp
from litellm.types.utils import CallTypes, ModelInfo, Usage
from litellm.utils import get_prompt_cache_min_tokens
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.utils import PrismaClient
from litellm.router import Router
_METADATA: Final = TypeAdapter(Mapping[str, object])
_PRICES: Final[TypeAdapter[ModelInfo | None]] = TypeAdapter(ModelInfo | None)
_JSON_BODY: Final = TypeAdapter(dict[str, JsonValue])
_COUNT_TIMEOUT: Final = 3.0
_MAX_COUNTS: Final = 4096
class CapturedBaselineObservation(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
scope: str
api_key: str
session_id: str
router_name: str
baseline_model: str
model: str
prices: ModelInfo | None
observation: BaselineObservation
@dataclass(frozen=True, slots=True)
class BaselineCacheContext:
collector: AutoRouterBaselineCache
capture: CapturedBaselineObservation
target: NativePredictionTarget | UnsupportedPredictionTarget
baseline_deployment_id: str
invalidated: str | None = None
class _Metadata(BaseModel):
model_config = ConfigDict(strict=True, arbitrary_types_allowed=True)
route: BaselineRouteStamp = Field(alias="_autorouter_baseline_route")
user_api_key_hash: str = Field(min_length=1)
session_id: str | None = None
class _WireEvent(BaseModel):
model_config = ConfigDict(strict=True, arbitrary_types_allowed=True)
httpx_response: httpx.Response
api_call_start_time: datetime
completion_start_time: datetime
custom_llm_provider: str
stream: bool = False
prompt_cache_response_complete: bool = False
class _ResponseUsage(BaseModel):
model_config = ConfigDict(strict=True, from_attributes=True)
usage: Usage | None = None
def _digest(value: object) -> str:
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
class AutoRouterBaselineCache(CustomLogger):
def __init__(
self,
prisma_client: PrismaClient | None,
router: Callable[[], Router | None] = _proxy_llm_router,
token_counter: TokenCounter | None = None,
clock: Callable[[], float] = time.time,
) -> None:
super().__init__() # pyright: ignore[reportUnknownMemberType] # legacy callback constructor
self.router: Final = router
self.token_counter: Final = token_counter
self.clock: Final = clock
self.count_slots: Final = asyncio.Semaphore(8)
self.counts: Mapping[str, tuple[int, float]] = MappingProxyType({})
async def async_pre_call_deployment_hook(self, kwargs: Mapping[str, object], call_type: CallTypes | None) -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
logging_obj: Final = kwargs.get("litellm_logging_obj")
if not isinstance(logging_obj, Logging) or call_type != CallTypes.anthropic_messages:
return
try:
metadata: Final = _METADATA.validate_python(
get_litellm_metadata_from_kwargs(
{"litellm_params": kwargs} # mutable-ok: legacy metadata owner requires a dictionary
)
)
if metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY):
return
if logging_obj.baseline_cache_context is not None:
await invalidate_baseline_cache(logging_obj, "retried_request")
return
request: Final = _Metadata.model_validate(metadata)
session: Final = kwargs.get("litellm_session_id") or request.session_id or logging_obj.litellm_session_id
if not isinstance(session, str) or not session or len(session) > 256:
return
router: Final = self.router()
deployment: Final = router.get_deployment(request.route.baseline_deployment_id) if router else None
if deployment is None:
return
target: Final = resolve_baseline_prediction_target(deployment.litellm_params)
prices: Final = _PRICES.validate_python(
_effective_model_info(router, request.route.baseline_deployment_id, request.route.baseline_model)
)
scope: Final = "autorouter-baseline:v3:" + _digest(
(
request.user_api_key_hash,
session,
request.route.router_name,
request.route.baseline_deployment_id,
deployment.litellm_params.model_dump(mode="json"),
prices,
)
)
started: Final = logging_obj.start_time.timestamp()
capture: Final = CapturedBaselineObservation(
scope=scope,
api_key=request.user_api_key_hash,
session_id=session,
router_name=request.route.router_name,
baseline_model=request.route.baseline_model,
model=target.model if isinstance(target, NativePredictionTarget) else request.route.baseline_model,
prices=prices,
observation=BaselineObservation(
request_id=logging_obj.litellm_call_id,
started_at=started,
available_at=started,
outcome="uncertain",
baseline_equivalent=False,
reason="incomplete_response",
),
)
logging_obj.baseline_cache_context = BaselineCacheContext(
self, capture, target, request.route.baseline_deployment_id
)
except Exception: # noqa: BLE001 # optional observation cannot fail inference
verbose_proxy_logger.warning("Auto-router baseline observation could not be initialized")
async def _count(self, target: NativePredictionTarget, body: Mapping[str, JsonValue]) -> int | None:
key: Final = _digest((target.model, target.api_key, target.api_base, _JSON_BODY.validate_python(body)))
now: Final = self.clock()
cached: Final = self.counts.get(key)
if cached is not None and cached[1] > now:
return cached[0]
async with self.count_slots:
tokens: Final = (
await self.token_counter(target.model, target.api_key, body)
if self.token_counter is not None
else await count_prompt_tokens(target.model, target.api_key, body, api_base=target.api_base)
)
if tokens is None or tokens < 0:
return None
retained: Final = tuple((k, v) for k, v in self.counts.items() if v[1] > now and k != key)[-(_MAX_COUNTS - 1) :]
self.counts = MappingProxyType(dict((*retained, (key, (tokens, now + 3600)))))
return tokens
async def plan(
self, target: NativePredictionTarget, wire: httpx.Request, body: Mapping[str, JsonValue], usage: Usage | None
) -> tuple[CountedPromptCachePlan | None, str | None]:
if not supported_prediction_headers(wire.headers):
return None, "unsupported_request_headers"
plan: Final = parse_cache_plan(body)
if isinstance(plan, UnsupportedCachePlan):
return None, plan.reason
details: Final = usage.prompt_tokens_details if usage is not None else None
if (
not plan.breakpoints
and details is not None
and ((details.cached_tokens or 0) + (details.cache_creation_tokens or 0))
):
return None, "implicit_cache_without_breakpoints"
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
return await self._count(target, body)
try:
counted: Final = await asyncio.wait_for(
count_cache_plan(target.model, target.api_key, plan, token_counter=count), timeout=_COUNT_TIMEOUT
)
return (None, counted.reason) if isinstance(counted, UnsupportedCachePlan) else (counted, None)
except TimeoutError:
return None, "token_count_timeout"
except Exception: # noqa: BLE001 # token counting cannot fail a completed request
return None, "token_count_unavailable"
async def invalidate_baseline_cache(logging_obj: Logging, reason: str, *, completed: bool = False) -> None:
context: Final = logging_obj.baseline_cache_context
if context is not None:
logging_obj.baseline_cache_context = replace(
context, invalidated=reason
) # rebind-ok: request-owned retry marker
logging_obj.baseline_observation = context.capture.model_copy(
update=MappingProxyType(
{ # rebind-ok: capture uncertainty for failure logging
"observation": context.capture.observation.model_copy(
update=MappingProxyType(
{
"available_at": max(context.capture.observation.started_at, context.collector.clock()),
"reason": reason,
}
)
),
}
)
)
async def finalize_baseline_cache(logging_obj: Logging, response_obj: object) -> None:
context: Final = logging_obj.baseline_cache_context
if context is None:
return
try:
capture: Final = await _capture(context, logging_obj, response_obj)
if logging_obj.baseline_cache_context is context:
logging_obj.baseline_observation = capture # rebind-ok: attach only to the captured request owner
except Exception: # noqa: BLE001 # observation failures must preserve inference and billing
await invalidate_baseline_cache(logging_obj, "observation_unavailable")
async def _capture(
context: BaselineCacheContext, logging_obj: Logging, response_obj: object
) -> CapturedBaselineObservation:
original: Final = context.capture.observation
details: Final = _METADATA.validate_python(logging_obj.model_call_details)
if details.get("cache_hit") is True:
return context.capture.model_copy(
update=MappingProxyType(
{
"observation": original.model_copy(
update=MappingProxyType({"outcome": "response_cache", "reason": "response_cache_hit"})
)
}
)
)
event: Final = _WireEvent.model_validate(details)
wire: Final = event.httpx_response.request
usage: Final = _ResponseUsage.model_validate(response_obj).usage
complete: Final = (
event.custom_llm_provider == "anthropic"
and event.httpx_response.status_code == 200
and (not event.stream or event.prompt_cache_response_complete)
)
started: Final = original.started_at
available: Final = event.completion_start_time.timestamp()
if context.invalidated or not complete or not started <= available <= context.collector.clock():
return context.capture.model_copy(
update=MappingProxyType(
{
"observation": original.model_copy(
update=MappingProxyType(
{
"available_at": max(started, context.collector.clock()),
"reason": context.invalidated or "incomplete_response",
}
)
)
}
)
)
target: Final = context.target
if isinstance(target, UnsupportedPredictionTarget) or not supported_baseline_recipient(target, wire):
return context.capture.model_copy(
update=MappingProxyType(
{
"observation": original.model_copy(
update=MappingProxyType(
{
"available_at": available,
"reason": target.reason
if isinstance(target, UnsupportedPredictionTarget)
else "unsupported_baseline_recipient",
}
)
)
}
)
)
body: Final = _JSON_BODY.validate_json(wire.content)
same: Final = (
logging_obj.get_router_model_id() == context.baseline_deployment_id and body.get("model") == target.model
)
plan, reason = await context.collector.plan(target, wire, body, usage)
minimum: Final = get_prompt_cache_min_tokens(target.model)
return context.capture.model_copy(
update=MappingProxyType(
{
"observation": BaselineObservation(
request_id=original.request_id,
started_at=started,
available_at=available,
outcome="complete",
baseline_equivalent=same,
usage=usage,
plan=plan,
minimum_cache_tokens=minimum,
reason=reason,
)
}
)
)

42
litellm/proxy/management_endpoints/auto_router_endpoints.py
View file

@ -556,6 +556,9 @@ class _SessionAggRow(BaseModel):
total_tokens: int
spend: float
saved_spend: float
savings_estimated_turns: int = 0
savings_estimated_actual_spend: float = 0.0
savings_estimated_saved_spend: float = 0.0
classifier_cost: float
classifier_cost_recorded_turns: int
session_seconds: float
@ -582,9 +585,19 @@ def _cache_bucket(turns: int, hits: int) -> AutoRouterCacheBucket:
return AutoRouterCacheBucket(turns=turns, hits=hits, hit_rate_pct=_pct(hits, turns))
def _savings_cohort(
turns: int, estimated_turns: int, actual_spend: float, saved_spend: float
) -> tuple[float | None, float | None]:
if turns > 0 and estimated_turns == 0:
return None, None
return saved_spend, actual_spend + saved_spend
def _benchmark_totals(row: _SessionAggRow) -> AutoRouterBenchmarkTotals:
return_misses: Final = row.return_turns - row.return_hits
baseline_spend: Final = row.spend + row.saved_spend
saved_spend, baseline_spend = _savings_cohort(
row.turns, row.savings_estimated_turns, row.savings_estimated_actual_spend, row.savings_estimated_saved_spend
)
sessions: Final = row.sessions
return AutoRouterBenchmarkTotals(
sessions=sessions,
@ -593,11 +606,15 @@ def _benchmark_totals(row: _SessionAggRow) -> AutoRouterBenchmarkTotals:
avg_session_seconds=row.session_seconds / sessions if sessions else 0.0,
avg_tokens_per_session=row.total_tokens / sessions if sessions else 0.0,
spend=row.spend,
saved_spend=row.saved_spend,
savings_estimated_turns=row.savings_estimated_turns,
savings_estimated_actual_spend=row.savings_estimated_actual_spend,
saved_spend=saved_spend,
classifier_cost=row.classifier_cost if row.classifier_cost_recorded_turns == row.turns else None,
baseline_spend=baseline_spend,
saved_pct=_pct(row.saved_spend, baseline_spend),
saved_per_session=row.saved_spend / sessions if sessions else 0.0,
saved_pct=_pct(saved_spend, baseline_spend) if saved_spend is not None and baseline_spend is not None else None,
saved_per_session=(row.savings_estimated_saved_spend / sessions if sessions else 0.0)
if row.savings_estimated_turns == row.turns
else None,
cache=AutoRouterCacheStats(
coverage_pct=_pct(row.covered_turns, row.turns),
hit_rate_pct=_pct(row.cache_hits, row.covered_turns),
@ -627,6 +644,8 @@ def _benchmark_group(row: _SessionAggRow) -> AutoRouterBenchmarkGroup:
avg_tokens_per_session=totals.avg_tokens_per_session,
spend=totals.spend,
saved_spend=totals.saved_spend,
savings_estimated_turns=totals.savings_estimated_turns,
savings_estimated_actual_spend=totals.savings_estimated_actual_spend,
classifier_cost=totals.classifier_cost,
baseline_spend=totals.baseline_spend,
saved_pct=totals.saved_pct,
@ -658,6 +677,9 @@ def _summed_agg_row(rows: Sequence[_SessionAggRow]) -> _SessionAggRow:
total_tokens=sum(row.total_tokens for row in rows),
spend=sum(row.spend for row in rows),
saved_spend=sum(row.saved_spend for row in rows),
savings_estimated_turns=sum(row.savings_estimated_turns for row in rows),
savings_estimated_actual_spend=sum(row.savings_estimated_actual_spend for row in rows),
savings_estimated_saved_spend=sum(row.savings_estimated_saved_spend for row in rows),
classifier_cost=sum(row.classifier_cost for row in rows),
classifier_cost_recorded_turns=sum(row.classifier_cost_recorded_turns for row in rows),
session_seconds=sum(row.session_seconds for row in rows),
@ -807,6 +829,9 @@ async def get_auto_router_session(
raise HTTPException(
status_code=404, detail=f"No auto-routed turns recorded for session {session_id!r} under this key"
)
saved_spend, baseline_spend = _savings_cohort(
row.turns, row.savings_estimated_turns, row.savings_estimated_actual_spend, row.savings_estimated_saved_spend
)
return AutoRouterSessionResponse(
session_id=session_id,
router_name=row.router_name,
@ -814,10 +839,13 @@ async def get_auto_router_session(
turns=row.turns,
last_model=row.last_model,
spend=row.spend,
saved_spend=row.saved_spend,
baseline_spend=row.spend + row.saved_spend,
savings_estimated_turns=row.savings_estimated_turns,
savings_estimated_actual_spend=row.savings_estimated_actual_spend,
saved_spend=saved_spend,
baseline_spend=baseline_spend if row.savings_estimated_turns == row.turns else None,
savings_estimated_baseline_spend=baseline_spend,
baseline_model=row.baseline_model,
baseline_models=row.baseline_models,
baseline_models=row.savings_estimated_baseline_models,
)

34
litellm/proxy/schema.prisma
View file

@ -1508,6 +1508,36 @@ model LiteLLM_AdaptiveRouterSession {
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
}
model LiteLLM_AutoRouterBaselineComparison {
scope String @id
api_key String
session_id String
router_name String
initial_equivalent Boolean
revision BigInt @default(0)
published_revision BigInt @default(0)
history String?
attempted_at DateTime?
retired Boolean @default(false)
updated_at DateTime @default(now())
@@index([api_key, session_id, router_name], map: "idx_autorouter_baseline_scope")
@@index([updated_at], map: "idx_autorouter_baseline_updated")
}
model LiteLLM_AutoRouterBaselineObservation {
request_id String @id
scope String
started_at Float
revision BigInt
data String
publication String?
conflicted Boolean @default(false)
@@index([scope, started_at, request_id], map: "idx_autorouter_baseline_event_order")
@@index([scope, revision, started_at], map: "idx_autorouter_baseline_event_revision")
}
model LiteLLM_AutoRouterSession {
api_key String
session_id String
@ -1534,6 +1564,10 @@ model LiteLLM_AutoRouterSession {
total_tokens BigInt @default(0)
spend Float @default(0)
saved_spend Float @default(0)
savings_estimated_turns Int @default(0)
savings_estimated_actual_spend Float @default(0)
savings_estimated_saved_spend Float @default(0)
savings_estimated_baseline_models Json @default("{}")
classifier_cost Float @default(0)
classifier_cost_recorded_turns Int @default(0)
tier_turns Json @default("{}")

348
litellm/proxy/spend_tracking/baseline_accounting.py Normal file
View file

@ -0,0 +1,348 @@
"""Pure, chronological cache accounting for the recorded baseline comparison.
Observation collection, pricing and durable publication belong to their existing
owners. Replaying these values in event order is independent of callback order.
"""
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass
from itertools import groupby
from math import isfinite
from types import MappingProxyType
from typing import Final, Literal
from pydantic import BaseModel, ConfigDict, Field
from litellm.llms.anthropic.prompt_cache_prediction import CountedBreakpoint, CountedPromptCachePlan
from litellm.types.utils import CacheCreationTokenDetails, PromptTokensDetailsWrapper, Usage
MAX_CACHE_TTL: Final = 3600
MAX_CACHE_ENTRIES: Final = 1024
class BaselineObservation(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
version: Literal[3] = 3
request_id: str = Field(min_length=1)
started_at: float = Field(allow_inf_nan=False, ge=0)
available_at: float = Field(allow_inf_nan=False, ge=0)
outcome: Literal["complete", "uncertain", "response_cache"]
baseline_equivalent: bool
usage: Usage | None = None
plan: CountedPromptCachePlan | None = None
minimum_cache_tokens: int = Field(default=0, ge=0)
reason: str | None = None
@dataclass(frozen=True, slots=True)
class BaselineEstimate:
request_id: str
reason: str
provenance: Literal["observed_identical", "modeled"] | None = None
usage: Usage | None = None
@dataclass(frozen=True, slots=True)
class CacheEntry:
fingerprint: str
content_fingerprint: str
tokens: int
ttl_seconds: int
available_at: float
expires_at: float
uncertain: bool = False
@dataclass(frozen=True, slots=True)
class BaselineHistory:
first_at: float | None = None
last_at: float | None = None
equivalent: bool = True
uncertain_before: float = 0.0
entries: tuple[CacheEntry, ...] = ()
blocked_until: float = 0.0
def _complete_usage(usage: Usage | None) -> bool:
if usage is None or usage.prompt_tokens < 0 or usage.completion_tokens < 0:
return False
details: Final = usage.prompt_tokens_details
if details is None:
return False
values: Final = (details.text_tokens, details.cached_tokens, details.cache_creation_tokens)
if any(value is None or value < 0 for value in values):
return False
split: Final = details.cache_creation_token_details
writes: Final = details.cache_creation_tokens or 0
return (
usage.total_tokens == usage.prompt_tokens + usage.completion_tokens
and sum(value or 0 for value in values) == usage.prompt_tokens
and (
writes == 0
or (
split is not None
and split.ephemeral_5m_input_tokens is not None
and split.ephemeral_1h_input_tokens is not None
and min(split.ephemeral_5m_input_tokens, split.ephemeral_1h_input_tokens) >= 0
and split.ephemeral_5m_input_tokens + split.ephemeral_1h_input_tokens == writes
)
)
)
def _valid_plan(plan: CountedPromptCachePlan | None) -> bool:
if plan is None or plan.total_tokens < 0 or len(plan.breakpoints) > 4:
return False
return all(
marker.fingerprint
and marker.content_fingerprint
and marker.fingerprint in marker.lookback_fingerprints
and marker.content_fingerprint in marker.lookback_content_fingerprints
and marker.ttl_seconds in (300, 3600)
and 0 <= marker.prefix_tokens <= plan.total_tokens
for marker in plan.breakpoints
) and all(
left.prefix_tokens <= right.prefix_tokens and left.ttl_seconds >= right.ttl_seconds
for left, right in zip(plan.breakpoints, plan.breakpoints[1:])
)
def _markers(observation: BaselineObservation) -> tuple[CountedBreakpoint, ...]:
return (
tuple(
marker
for marker in observation.plan.breakpoints
if marker.prefix_tokens >= observation.minimum_cache_tokens
)
if observation.plan is not None
else ()
)
def _matches(entry: CacheEntry, markers: tuple[CountedBreakpoint, ...], started: float) -> bool:
return entry.available_at <= started < entry.expires_at and any(
entry.fingerprint in marker.lookback_fingerprints
and entry.tokens <= marker.prefix_tokens
and entry.ttl_seconds == marker.ttl_seconds
for marker in markers
)
def _ambiguous(entry: CacheEntry, markers: tuple[CountedBreakpoint, ...], started: float) -> bool:
return entry.available_at <= started < entry.expires_at and any(
entry.content_fingerprint in marker.lookback_content_fingerprints
and (entry.uncertain or entry.ttl_seconds != marker.ttl_seconds)
for marker in markers
)
def _usage_with_cache(usage: Usage, total: int, read: int, write_5m: int, write_1h: int) -> Usage:
writes: Final = write_5m + write_1h
original_details: Final = usage.prompt_tokens_details or PromptTokensDetailsWrapper()
details: Final = original_details.model_copy(
deep=True,
update=MappingProxyType(
{
"text_tokens": total - read - writes,
"cached_tokens": read,
"cache_creation_tokens": writes,
"cache_write_tokens": writes,
"cache_creation_token_details": CacheCreationTokenDetails(
ephemeral_5m_input_tokens=write_5m,
ephemeral_1h_input_tokens=write_1h,
),
}
),
)
return Usage.model_validate(
{ # mutable-ok: Usage only runs its normalizing constructor for a plain dictionary
**usage.model_dump(),
"prompt_tokens": total,
"total_tokens": total + usage.completion_tokens,
"prompt_tokens_details": details,
"cache_read_input_tokens": read,
"cache_creation_input_tokens": writes,
},
)
def _estimate(history: BaselineHistory, observation: BaselineObservation, equivalent: bool) -> BaselineEstimate:
if observation.outcome != "complete" or not _complete_usage(observation.usage):
return BaselineEstimate(observation.request_id, observation.reason or observation.outcome)
usage: Final = observation.usage
if usage is None:
return BaselineEstimate(observation.request_id, "missing_usage")
if equivalent and observation.baseline_equivalent:
return BaselineEstimate(
observation.request_id, "identical_baseline_path", "observed_identical", usage.model_copy(deep=True)
)
if observation.started_at < history.blocked_until:
return BaselineEstimate(observation.request_id, "concurrent_uncertainty")
plan: Final = observation.plan
if not _valid_plan(plan) or plan is None:
return BaselineEstimate(observation.request_id, observation.reason or "unsupported_cache_plan")
markers: Final = _markers(observation)
if any(_ambiguous(entry, markers, observation.started_at) for entry in history.entries):
return BaselineEstimate(observation.request_id, "cache_ttl_changed")
read: Final = max(
(
entry.tokens
for entry in history.entries
if not entry.uncertain and _matches(entry, markers, observation.started_at)
),
default=0,
)
end: Final = markers[-1].prefix_tokens if markers else 0
if read < end and observation.started_at < history.uncertain_before + max(marker.ttl_seconds for marker in markers):
return BaselineEstimate(observation.request_id, "history_unavailable")
one_hour: Final = max(
(marker.prefix_tokens for marker in markers if marker.ttl_seconds == 3600 and marker.prefix_tokens > read),
default=read,
)
expired: Final = any(
entry.expires_at <= observation.started_at
and any(entry.fingerprint in marker.lookback_fingerprints for marker in markers)
for entry in history.entries
)
reason: Final = (
"cache_prefix_available"
if read
else "cache_prefix_expired"
if expired
else "cache_prefix_cold"
if markers
else "below_cache_minimum"
if plan.breakpoints
else "no_cache_breakpoints"
)
return BaselineEstimate(
observation.request_id,
reason,
"modeled",
_usage_with_cache(usage, plan.total_tokens, read, end - one_hour, one_hour - read),
)
def _writes(history: BaselineHistory, observation: BaselineObservation) -> tuple[CacheEntry, ...]:
if (
observation.outcome != "complete"
or observation.started_at < history.blocked_until
or not _complete_usage(observation.usage)
or not _valid_plan(observation.plan)
):
return ()
markers: Final = _markers(observation)
ambiguous: Final = tuple(entry for entry in history.entries if _ambiguous(entry, markers, observation.started_at))
hit: Final = (
max(
(
entry
for entry in history.entries
if not entry.uncertain and _matches(entry, markers, observation.started_at)
),
key=lambda entry: entry.tokens,
default=None,
)
if not ambiguous
else None
)
refresh: Final = (
(
CacheEntry(
hit.fingerprint,
hit.content_fingerprint,
hit.tokens,
hit.ttl_seconds,
observation.available_at,
observation.started_at + hit.ttl_seconds,
),
)
if hit is not None and all(marker.fingerprint != hit.fingerprint for marker in markers)
else ()
)
return (
*refresh,
*(
CacheEntry(
marker.fingerprint,
marker.content_fingerprint,
marker.prefix_tokens,
marker.ttl_seconds,
observation.available_at,
observation.started_at + max((marker.ttl_seconds, *(entry.ttl_seconds for entry in ambiguous))),
uncertain=bool(ambiguous),
)
for marker in markers
),
)
def _entry_key(entry: CacheEntry) -> tuple[str, str, int, int, bool]:
return entry.fingerprint, entry.content_fingerprint, entry.tokens, entry.ttl_seconds, entry.uncertain
def _compact_entries(entries: tuple[CacheEntry, ...], started: float) -> tuple[CacheEntry, ...]:
ordered: Final = sorted((entry for entry in entries if entry.expires_at >= started - MAX_CACHE_TTL), key=_entry_key)
return tuple(
retained
for _, values in groupby(ordered, key=_entry_key)
for group in (tuple(values),)
for retained in (
max(
(entry for entry in group if entry.available_at <= started),
key=lambda entry: entry.expires_at,
default=None,
),
*(entry for entry in group if entry.available_at > started),
)
if retained is not None
)
def advance_baseline_history(
history: BaselineHistory,
simultaneous: Sequence[BaselineObservation],
) -> tuple[BaselineHistory, tuple[BaselineEstimate, ...]]:
"""Apply one request-start timestamp; ties cannot manufacture initial equality.
The storage owner groups and orders observations before calling this function.
Equal timestamps are evaluated against the same preceding cache snapshot.
"""
if not simultaneous:
return history, ()
started: Final = simultaneous[0].started_at
valid_order: Final = (
isfinite(started)
and all(item.started_at == started and item.available_at >= started for item in simultaneous)
and (history.last_at is None or started > history.last_at)
)
if not valid_order:
return history, tuple(BaselineEstimate(item.request_id, "invalid_observation_order") for item in simultaneous)
first: Final = started if history.first_at is None else history.first_at
uncertain: Final = max(history.uncertain_before, first)
relevant: Final = tuple(item for item in simultaneous if item.outcome != "response_cache")
equivalent: Final = history.equivalent and all(item.baseline_equivalent for item in relevant)
before: Final = BaselineHistory(
first, history.last_at, equivalent, uncertain, history.entries, history.blocked_until
)
estimates: Final = tuple(_estimate(before, item, equivalent) for item in simultaneous)
invalidated: Final = any(
item.outcome != "complete" or not _complete_usage(item.usage) or not _valid_plan(item.plan) for item in relevant
)
blocked: Final = max((history.blocked_until, *(item.available_at for item in relevant if invalidated)))
entries: Final = _compact_entries(
() if invalidated else (*history.entries, *(entry for item in relevant for entry in _writes(before, item))),
started,
)
overflow: Final = len(entries) > MAX_CACHE_ENTRIES
return BaselineHistory(
first_at=first,
last_at=started,
equivalent=equivalent,
uncertain_before=max(started, blocked) if invalidated or overflow else uncertain,
entries=() if overflow else entries,
blocked_until=blocked,
), estimates

266
litellm/proxy/spend_tracking/savings.py
View file

@ -10,7 +10,11 @@ have been aggregated across models.
from collections.abc import Callable, Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Final, NamedTuple
from math import isclose, isfinite
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, NamedTuple
from pydantic import BaseModel, ConfigDict, Field
import litellm
from litellm._logging import verbose_proxy_logger
@ -65,7 +69,7 @@ def _resolve_model(model: str | None, custom_llm_provider: str | None) -> _Model
return None
try:
resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
except Exception as e: # noqa: BLE001 # get_llm_provider raises for unroutable names; degrade to zero savings
except Exception as e: # noqa: BLE001 # get_llm_provider raises for unroutable names; degrade to an unavailable estimate
verbose_proxy_logger.debug(
"savings: cannot resolve provider for model=%s custom_llm_provider=%s (%s)", model, custom_llm_provider, e
)
@ -118,6 +122,68 @@ class PricingBasis(NamedTuple):
_STANDARD_RATES: Final = PricingBasis()
class BaselineCostSnapshot(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
model: str
provider: str
prices: ModelInfo | None
basis: PricingBasis = _STANDARD_RATES
actual_spend: float = Field(allow_inf_nan=False, ge=0)
actual_token_cost: float | None = Field(default=None, allow_inf_nan=False, ge=0)
classifier_cost: float = Field(default=0.0, allow_inf_nan=False, ge=0)
def baseline_cost_snapshot(
model: str,
prices: ModelInfo | None,
actual_spend: float,
cost_breakdown: Mapping[str, object] | None,
routing_decision: Mapping[str, object] | None,
) -> BaselineCostSnapshot:
return BaselineCostSnapshot(
model=model,
provider="anthropic",
prices=prices,
actual_spend=actual_spend,
basis=_pricing_basis(cost_breakdown),
actual_token_cost=_recorded_token_cost(cost_breakdown),
classifier_cost=classifier_cost_from_decision(routing_decision) or 0.0,
)
class BaselineCosts(NamedTuple):
actual: float
baseline: float
@property
def savings(self) -> float:
return self.baseline - self.actual
def price_baseline_comparison(
snapshot: BaselineCostSnapshot,
baseline_usage: Usage | None,
provenance: Literal["observed_identical", "modeled"] | None,
) -> BaselineCosts | None:
if baseline_usage is None or provenance is None:
return None
actual: Final = snapshot.actual_spend + snapshot.classifier_cost
if provenance == "observed_identical":
return BaselineCosts(actual=actual, baseline=snapshot.actual_spend)
if snapshot.prices is None or snapshot.actual_token_cost is None:
return None
token_cost: Final = _cost_of_usage(
_ModelIdentity(snapshot.model, snapshot.provider), baseline_usage, snapshot.prices, snapshot.basis
)
if token_cost is None or not isfinite(token_cost) or token_cost < 0:
return None
baseline: Final = snapshot.actual_spend + token_cost - snapshot.actual_token_cost
if not isfinite(baseline) or baseline < 0:
return None
return BaselineCosts(actual=actual, baseline=baseline)
def _pricing_basis(cost_breakdown: Mapping[str, object] | None) -> PricingBasis:
"""The basis recorded on a request, defaulting to standard rates when absent.
@ -225,56 +291,16 @@ def _baseline_cache_rate_keys(baseline_info: ModelInfo | None) -> tuple[bool, bo
)
def _baseline_usage(usage: Usage, conversation_continuing: bool, baseline_info: ModelInfo | None = None) -> Usage:
"""The same request as a single-model baseline would have met it.
The baseline is one model serving every turn, so whether it had this prompt cached
is simply whether the conversation was already underway. On a continuing
conversation it wrote the prompt on an earlier turn and would only read it now, so
the cache tokens move into the read bucket and whatever this request paid to write
counts against the saving; that write is what switching models costs.
On a conversation's first turn nothing was cached anywhere, for any model. The
baseline would have written the same prompt, so the cache buckets stay where they are
and both arms carry the write at their own rates, unless the baseline has no rate for
a bucket, in which case those tokens are its plain input. Charging the write to this case
too, which is all a single rollup row can support, understates a first turn to a
few percent of its value and can render a profitable route as a loss.
A continuing turn that mostly read from cache is the third case: the selected model
was already warm, so it is the one that has been serving this conversation and the
baseline's cache holds exactly what its does. The tokens written are the turn's own
growth, new to every model, and the baseline would have paid to write them too.
Moving them would forgive the baseline a write it really owes and shrink the
reported saving. "Mostly read" rather than "read anything" on purpose: a switch onto
a model holding a small prefix of this prompt still writes most of it, and must keep
counting that write against the saving.
Only the cache buckets move. Every other field the request was priced on travels
through untouched, audio and image and video counts among them, because the baseline
is this same request served by a model that happened to be warm; naming the fields to
keep instead would price the baseline on a request that never ran, and would go stale
the next time a priced field is added.
"""
def _baseline_usage(usage: Usage, baseline_info: ModelInfo | None = None) -> Usage:
cache_read, cache_creation = _cache_token_split(usage)
details: Final = usage.prompt_tokens_details
if details is None or (cache_read <= 0 and cache_creation <= 0):
return usage
# The tokens this request paid to write move into the cached count and the creation
# charge is dropped: on one model that cache was already warm, so the baseline would
# have read them rather than paying to create them. The 5m/1h breakdown goes with
# them; left behind it re-charges the write.
warm: Final = conversation_continuing and cache_creation > 0 and cache_read <= cache_creation
reads = cache_read + cache_creation if warm else cache_read
writes = 0 if warm else cache_creation
prices_reads, prices_writes = _baseline_cache_rate_keys(baseline_info)
reads = reads if prices_reads else 0
writes = writes if prices_writes else 0
reads: Final = cache_read if prices_reads else 0
writes: Final = cache_creation if prices_writes else 0
if (reads, writes) == (cache_read, cache_creation):
return usage
other_modalities: Final = sum(
(getattr(details, field, 0) or 0) for field in ("audio_tokens", "image_tokens", "video_tokens")
)
@ -309,64 +335,47 @@ def compute_autorouter_savings(
cost_breakdown: Mapping[str, object] | None = None,
baseline_deployment_id: str | None = None,
selected_deployment_id: str | None = None,
) -> float:
"""Net dollars the router saved, or cost, by serving this request on ``selected_model``.
Signed on purpose. Switching models leaves the new one with a cold cache, so the
request pays a cache-creation charge that staying on one model would not have
incurred; when that charge outweighs the cheaper rates, routing lost money and the
dashboard has to be able to say so. Zero when both sides resolve to the same
deployment, or when either cannot be resolved or priced.
Only one side of this subtraction is a counterfactual. What the request cost on the
model that served it is a number the operator was actually billed, and the cost
calculator already wrote it down, so ``cost_breakdown`` is read rather than
re-derived. Recomputing it means restating every pricing dimension the biller
applied, and each one omitted is a silent disagreement with the ``spend`` column
beside it; a request billed at a priority tier recomputed at standard rates reads as
half its real cost.
The baseline has no such record, since it never ran, so it is priced through the same
cost engine on the basis the biller used for this request. An operator running that
one model instead of the router would have sent this request to the same tier and the
same region, because both are properties of the request and the deployment's
contract, not of which model the router happened to pick.
``conversation_continuing`` says whether the baseline would already have had this
prompt cached. It defaults to True because that is the conservative reading: a
request whose shape the router could not determine is charged the write and
under-claims rather than inflating a savings figure.
"""
# No provider argument for the baseline on purpose: it arrives from the routing
# metadata as a single self-describing string, already qualified by the auto-router,
# so there is no second field that could disagree with it.
baseline_usage: Usage | None = None,
baseline_provenance: Literal["observed_initial", "modeled"] | None = None,
) -> float | None:
"""Price established baseline usage; conversation shape cannot establish cache warmth."""
baseline: Final = _resolve_model(baseline_model, None)
selected: Final = _resolve_model(selected_model, selected_provider)
if baseline is None or selected is None:
return 0.0
same_target: Final = (
baseline_deployment_id == selected_deployment_id
if baseline_deployment_id and selected_deployment_id
else baseline == selected
)
if same_target:
return 0.0
return None
if baseline_usage is None and any(_cache_token_split(usage)):
return None
basis: Final = _pricing_basis(cost_breakdown)
effective_baseline_info: Final = baseline_info if baseline_info is not None else _model_info(baseline)
modeled_usage: Final = baseline_usage if baseline_usage is not None else usage
baseline_cost: Final = _cost_of_usage(
baseline,
_baseline_usage(usage, conversation_continuing, effective_baseline_info),
effective_baseline_info,
basis,
baseline, _baseline_usage(modeled_usage, effective_baseline_info), effective_baseline_info, basis
)
recorded_selected_cost: Final = _recorded_token_cost(cost_breakdown)
selected_cost: Final = (
recorded_selected_cost
if recorded_selected_cost is not None
else _cost_of_usage(selected, usage, selected_info, basis)
)
# Falls back to pricing the request only when the biller recorded nothing, which is
# every row written before the breakdown carried its basis.
selected_cost = _recorded_token_cost(cost_breakdown)
if selected_cost is None:
selected_cost = _cost_of_usage(selected, usage, selected_info, basis)
if baseline_cost is None or selected_cost is None:
return 0.0
return baseline_cost - selected_cost
return None
if baseline_provenance == "observed_initial":
same_prices: Final = effective_baseline_info == (
selected_info if selected_info is not None else _model_info(selected)
)
equivalent: Final = (
baseline_usage is not None
and baseline_usage == usage
and baseline == selected
and bool(baseline_deployment_id)
and baseline_deployment_id == selected_deployment_id
and same_prices
and recorded_selected_cost is not None
and isclose(baseline_cost, recorded_selected_cost, rel_tol=1e-9, abs_tol=1e-12)
)
return 0.0 if equivalent else None
difference: Final = baseline_cost - selected_cost
return difference if isfinite(difference) else None
def _usage_from_spend_log(usage_object: Mapping[str, object] | None) -> Usage | None:
@ -463,11 +472,23 @@ def _proxy_llm_router() -> "Router | None":
def _numeric_savings(value: object) -> float | None:
"""``value`` as a recorded savings figure, or ``None`` when it is not one."""
if isinstance(value, bool) or not isinstance(value, (int, float)):
if isinstance(value, bool) or not isinstance(value, (int, float)) or not isfinite(value):
return None
return float(value)
def recorded_estimated_autorouter_savings(metadata: Mapping[str, object]) -> float | None:
estimate: Final = metadata.get("autorouter_savings_estimate")
if (
not isinstance(estimate, Mapping)
or type(estimate.get("version")) is not int
or estimate.get("version") not in (1, 2, 3)
or estimate.get("status") != "estimated"
):
return None
return _numeric_savings(metadata.get("autorouter_savings"))
def classifier_cost_from_decision(routing_decision: Mapping[str, object] | None) -> float | None:
"""The LLM-classifier cost a routing decision recorded, or ``None`` when it holds none.
@ -490,22 +511,10 @@ def autorouter_savings_for_request(
model_id: str | None = None,
llm_router: "Callable[[], Router | None] | None" = None,
cost_breakdown: Mapping[str, object] | None = None,
baseline_usage: Usage | None = None,
baseline_provenance: Literal["observed_initial", "modeled"] | None = None,
) -> float | None:
"""Auto-router savings for one request, net of the classifier call that routed it,
or ``None`` when the driver is off.
``None`` and ``0.0`` are different facts: ``None`` means this request cannot carry a
figure at all (no routing decision, no baseline, unusable usage), while ``0.0`` is a
real figure for a routed request whose baseline resolved to the served deployment.
Never raises: pricing failures inside degrade to zero, and the driver-off cases
return ``None``, so this is safe on the logging path where a raise would fail the
request's logging.
The classifier deduction lives here, at the figure's one computation owner, rather
than in any reader: the stamped ``autorouter_savings`` is then already net, so the
session rollup, the daily tables and every logging consumer agree without each
re-deriving the deduction, and the recorded-figure-wins path cannot deduct twice.
"""
"""Return net savings for established usage, or None when the estimate is unavailable."""
usage: Final = _usage_from_spend_log(usage_object)
if usage is None or not model:
return None
@ -522,15 +531,16 @@ def autorouter_savings_for_request(
selected_model=model,
selected_provider=custom_llm_provider,
usage=usage,
# Absent means the router never recorded a shape, which is the conservative
# reading: charge the cache write rather than claim a first turn's saving.
conversation_continuing=decision.get("conversation_continuing") is not False,
selected_info=_effective_model_info(router_instance, model_id, model or ""),
baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""),
cost_breakdown=cost_breakdown,
baseline_deployment_id=baseline_id,
selected_deployment_id=model_id,
baseline_usage=baseline_usage,
baseline_provenance=baseline_provenance,
)
if gross is None:
return None
classifier_cost: Final = classifier_cost_from_decision(decision)
return gross if classifier_cost is None else gross - classifier_cost
@ -542,6 +552,8 @@ def autorouter_savings_for_logging_payload(
model_id: str | None,
usage_object: Mapping[str, object] | None,
cost_breakdown: Mapping[str, object] | None,
baseline_usage: Usage | None = None,
baseline_provenance: Literal["observed_initial", "modeled"] | None = None,
) -> float | None:
"""The figure the logging payload records for a request, or ``None`` when none should be.
@ -561,6 +573,8 @@ def autorouter_savings_for_logging_payload(
model_id=model_id,
llm_router=_proxy_llm_router,
cost_breakdown=cost_breakdown,
baseline_usage=baseline_usage,
baseline_provenance=baseline_provenance,
)
@ -575,6 +589,7 @@ def compute_savings_spend(
llm_router: "Callable[[], Router | None] | None" = None,
cost_breakdown: Mapping[str, object] | None = None,
recorded_autorouter_savings: object = None,
recorded_autorouter_savings_estimate: Mapping[str, object] | None = None,
billed_at: datetime | str | None = None,
) -> SavingsSpend:
"""
@ -604,11 +619,9 @@ def compute_savings_spend(
figure is normally the smaller of the two, being a subset of the same requests, but
not always: a request that only writes cache and never reads it has negative net
savings, and dropping such a request from the attributed figure can lift it above
the total. Auto-router savings compare the
served ``model`` against the counterfactual baseline the router recorded on
its ``routing_decision``, and are zero unless the two differ. That record
also says whether the conversation was already underway, which is what tells
a mid-conversation switch from a first turn.
the total. Auto-router savings compare established baseline usage against the
recorded selected-model cost. Versioned unknown estimates contribute no dollars
to this subtotal and are excluded from the separately reported coverage cohort.
``llm_router`` is passed as a provider rather than a router because every spend write
calls this and only auto-routed ones need one, so looking it up eagerly at the call
@ -653,10 +666,21 @@ def compute_savings_spend(
# The figure the logging path recorded wins, before the usage gate on purpose: a row
# whose usage no longer parses still carries the number computed when it did.
recorded_savings: Final = _numeric_savings(recorded_autorouter_savings)
recorded_savings: Final = (
recorded_estimated_autorouter_savings(
MappingProxyType(
{
"autorouter_savings": recorded_autorouter_savings,
"autorouter_savings_estimate": recorded_autorouter_savings_estimate,
}
)
)
if recorded_autorouter_savings_estimate is not None
else _numeric_savings(recorded_autorouter_savings)
)
autorouter: Final = (
recorded_savings
if recorded_savings is not None
if recorded_savings is not None or recorded_autorouter_savings_estimate is not None
else autorouter_savings_for_request(
model=model,
custom_llm_provider=custom_llm_provider,

34
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -7,7 +7,7 @@ from datetime import datetime as dt
from types import MappingProxyType
from typing import Final, Literal, Protocol, cast, runtime_checkable
from pydantic import BaseModel
from pydantic import BaseModel, JsonValue
import litellm
from litellm._logging import verbose_proxy_logger
@ -143,6 +143,8 @@ def _get_spend_logs_metadata(
cost_breakdown: CostBreakdown | None = None,
litellm_call_id: str | None = None,
autorouter_savings: float | None = None,
autorouter_savings_estimate: Mapping[str, JsonValue] | None = None,
autorouter_baseline_observation: str | None = None,
router_metadata: SpendLogsRouterMetadata | None = None,
) -> SpendLogsMetadata:
if metadata is None:
@ -182,6 +184,8 @@ def _get_spend_logs_metadata(
cost_breakdown=None,
compression_savings=None,
autorouter_savings=autorouter_savings,
autorouter_savings_estimate=autorouter_savings_estimate,
autorouter_baseline_observation=autorouter_baseline_observation,
litellm_gateway_injected_cache=None,
litellm_call_id=litellm_call_id,
router_metadata=router_metadata,
@ -192,7 +196,22 @@ def _get_spend_logs_metadata(
# Filter the metadata dictionary to include only the specified keys
clean_metadata: Final = SpendLogsMetadata(
**{key: metadata.get(key) for key in SpendLogsMetadata.__annotations__ if key != "router_metadata"},
**MappingProxyType(
{
key: metadata.get(key)
for key in SpendLogsMetadata.__annotations__
if key
not in (
"router_metadata",
"autorouter_savings",
"autorouter_savings_estimate",
"autorouter_baseline_observation",
)
}
),
autorouter_savings=autorouter_savings,
autorouter_savings_estimate=autorouter_savings_estimate,
autorouter_baseline_observation=autorouter_baseline_observation,
router_metadata=router_metadata,
)
_raw_key: Final = clean_metadata.get("user_api_key")
@ -215,7 +234,6 @@ def _get_spend_logs_metadata(
clean_metadata["cold_storage_object_key"] = cold_storage_object_key
clean_metadata["litellm_overhead_time_ms"] = litellm_overhead_time_ms
clean_metadata["cost_breakdown"] = cost_breakdown
clean_metadata["autorouter_savings"] = autorouter_savings
clean_metadata["litellm_call_id"] = litellm_call_id
return clean_metadata
@ -522,6 +540,16 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
autorouter_savings=(
standard_logging_payload.get("autorouter_savings", None) if standard_logging_payload is not None else None
),
autorouter_savings_estimate=(
standard_logging_payload.get("autorouter_savings_estimate")
if standard_logging_payload is not None
else None
),
autorouter_baseline_observation=(
standard_logging_payload.get("autorouter_baseline_observation")
if standard_logging_payload is not None
else None
),
litellm_call_id=litellm_call_id,
router_metadata=_get_router_metadata_for_spend_log(
metadata=metadata,

28
litellm/proxy/utils.py
View file

@ -219,6 +219,7 @@ if TYPE_CHECKING:
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.models.team import LiteLLM_TeamTableCachedObj
from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
from litellm.proxy.db.baseline_accounting import BaselineAccountingRecord
from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction
from litellm.repositories.prisma_protocols import TableActions
from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline
@ -2920,6 +2921,10 @@ class ProxyLogging:
Otherwise, returns None and the original exception is used.
"""
logging_obj: Final[object] = request_data.get("litellm_logging_obj") # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] # legacy request data is narrowed to Logging below
if isinstance(logging_obj, Logging) and logging_obj.baseline_cache_context is not None:
await logging_obj.invalidate_baseline_cache_estimate("failed_request", completed=True)
### ALERTING ###
await self.update_request_status(litellm_call_id=request_data.get("litellm_call_id", ""), status="fail")
if AlertType.llm_exceptions in self.alert_types and not _is_client_error_exception(original_exception):
@ -4067,6 +4072,10 @@ class PrismaClient:
http_client: "HttpConfig | None" = None,
):
## init logging object
self.baseline_accounting_transactions: list[
BaselineAccountingRecord
] = [] # mutable-ok: locked background queue
self.baseline_accounting_lock: Final = asyncio.Lock()
self.proxy_logging_obj = proxy_logging_obj
self.token_auth: DatabaseTokenAuth | None = resolve_database_token_auth()
verbose_proxy_logger.debug("Creating Prisma Client..")
@ -7033,7 +7042,15 @@ async def _total_queued_spend_transactions(prisma_client: PrismaClient) -> int:
autorouter_queue_size: Final = len(prisma_client.autorouter_turn_transactions)
from litellm.proxy.db.shadow_eval_funnel import pending_shadow_eval_funnel_events
return spend_queue_size + tool_queue_size + autorouter_queue_size + pending_shadow_eval_funnel_events()
async with prisma_client.baseline_accounting_lock:
baseline_queue_size: Final = len(prisma_client.baseline_accounting_transactions)
return (
spend_queue_size
+ tool_queue_size
+ autorouter_queue_size
+ baseline_queue_size
+ pending_shadow_eval_funnel_events()
)
async def update_daily_tag_spend(
@ -7098,7 +7115,10 @@ async def update_spend_logs_job(
# Atomically pop batch from queue. The tool usage queue counts toward the
# emptiness check: a spend-log write failure aborts a run before the tool
# drain below, and those entries must not strand once the spend queue drains.
from litellm.proxy.db.baseline_accounting import flush_baseline_accounting
if await _total_queued_spend_transactions(prisma_client) == 0:
await flush_baseline_accounting(prisma_client)
return
logs_to_process: Final = await dequeue_spend_logs(prisma_client, MAX_LOGS_PER_INTERVAL)
@ -7155,6 +7175,8 @@ async def update_spend_logs_job(
tool_tracking_err,
)
await flush_baseline_accounting(prisma_client)
async with prisma_client._autorouter_turn_transactions_lock:
autorouter_turns_to_process: Final = prisma_client.autorouter_turn_transactions[:MAX_LOGS_PER_INTERVAL]
remaining_autorouter_turns: Final = prisma_client.autorouter_turn_transactions[
@ -7277,7 +7299,9 @@ async def _monitor_spend_logs_queue(
proxy_logging_obj=proxy_logging_obj,
)
else:
# Exponential backoff when no logs to process
from litellm.proxy.db.baseline_accounting import flush_baseline_accounting
await flush_baseline_accounting(prisma_client)
current_interval = min(current_interval * backoff_multiplier, max_backoff)
if await _wait_for_spend_log_flush_request(flush_requested, current_interval):

14
litellm/router.py
View file

@ -13752,6 +13752,20 @@ class Router:
to the deployment that actually served the request. Every attempt therefore
writes or clears, never just writes.
"""
from litellm.types.router import BaselineRouteStamp
baseline_model: Final = routing_decision.get("savings_baseline_model") if routing_decision else None
baseline_id: Final = routing_decision.get("savings_baseline_deployment_id") if routing_decision else None
router_name: Final = routing_decision.get("router_model_name") if routing_decision else None
Router._stamp_or_clear_metadata_key(
request_kwargs=request_kwargs,
key="_autorouter_baseline_route",
value=(
BaselineRouteStamp(router_name, baseline_model, baseline_id)
if router_name and baseline_model and baseline_id
else None
),
)
Router._stamp_or_clear_metadata_key(
request_kwargs=request_kwargs,
key="routing_decision",

19
litellm/router_strategy/complexity_router/README.md
View file

@ -11,6 +11,25 @@ Unlike the semantic `auto_router` which uses embedding-based matching, the `comp
- **Predictable behavior** - rule-based scoring is deterministic
- **Fully configurable** - weights, thresholds, and keyword lists can be customized
## Savings estimates
The Cost Optimization dashboard compares routed spend with an estimate of sending the same requests to the configured highest-tier baseline model. Actual spend includes recorded classifier costs. A negative estimate can reflect real cache-write costs when switching models, even when the selected model has cheaper token prices
For supported native Anthropic `/v1/messages` requests, a baseline read requires a matching prefix that was available when the request started and remained inside its five-minute or one-hour TTL. Requests served by cheaper models advance the hypothetical baseline history too. An assistant message alone never establishes a cache hit
Use a stable session ID and a configured proxy database. Accounting runs after inference, through the background spend pipeline. The primary database retains compact observations and replays them in request-start order. Late arrivals withdraw affected estimates until replay publishes corrected per-request, session and daily totals. Actual billed spend remains unchanged
Before any recorded request diverges from the exact baseline deployment, complete observed usage establishes equal nonzero model costs and zero model savings, including overlapping requests. Recorded classifier cost contributes only to actual routed spend. After divergence, comparisons require modeled cache evidence even when routing returns to the baseline. Missing history, unsupported cache semantics and incomplete requests produce unavailable estimates
The comparison holds recorded prompts, output usage, request-start times and first-token times fixed. It does not predict alternate model responses, provider evictions or unrecorded traffic. Legacy sessions do not acquire an initial observed estimate merely because the observation journal is empty. Session retention retires inactive comparisons and prunes their observations; retained retirement markers prevent a reused session from acquiring another initial estimate
Baseline counting requires the same endpoint and API key as the served request. Configured Anthropic-compatible gateways must support native counting for every required prefix, including system/tools-only prefixes with empty `messages`. Missing counts remain unknown; local tokenizers, synthetic messages and partial counts cannot establish a cache hit
Some Claude Code beta headers and `context_management` shapes remain unsupported for modeled cache accounting. Initial baseline-identical requests can still use complete observed usage. Support after a model switch depends on the actual request shape and available prefix counts
Spend metadata records a versioned `autorouter_savings_estimate` with comparison identity, provenance, status, reason and both costs. The dashboard compares costs over the same estimated turns, including numeric zero savings, and shows coverage alongside total actual spend. Pending estimates contribute to neither comparison cost. Existing session-status clients receive no baseline total when coverage is partial
## How It Works
The router scores each request across 7 dimensions:

38
litellm/types/management_endpoints/auto_router_endpoints.py
View file

@ -199,13 +199,20 @@ class AutoRouterBenchmarkTotals(BaseModel):
description="Recorded LLM classifier cost already included in spend; null when any session turns predate "
"subtotal recording, and zero for an empty window"
)
saved_spend: float = Field(
description="Signed dollars saved versus each router's savings baseline (derived from its hardest "
"tier, or the configured override), from the same per-request savings record the usage tab reads"
savings_estimated_turns: int = Field(
description="Turns covered by the current savings estimator; legacy estimates are excluded"
)
savings_estimated_actual_spend: float = Field(
description="Actual spend, including classifier cost, for covered turns only"
)
saved_spend: float | None = Field(
description="Signed savings for covered turns only; null when traffic has no current estimates"
)
baseline_spend: float | None = Field(description="Estimated single-model cost for covered turns only")
saved_pct: float | None = Field(description="Covered savings over covered baseline spend, as a percentage")
saved_per_session: float | None = Field(
description="Average session savings; unavailable unless every turn is covered"
)
baseline_spend: float = Field(description="spend plus saved_spend: the estimated single-model cost")
saved_pct: float = Field(description="saved_spend over baseline_spend, as a percentage")
saved_per_session: float
cache: AutoRouterCacheStats
@ -236,16 +243,27 @@ class AutoRouterSessionResponse(BaseModel):
turns: int = Field(description="Auto-routed turns the rollup has recorded for this session so far")
last_model: str = Field(description="The deployment model the most recent turn was routed to")
spend: float = Field(description="What the session's routed traffic actually cost, classifier calls included")
saved_spend: float = Field(description="Estimated savings against the baseline, net of classifier cost")
baseline_spend: float = Field(description="spend plus saved_spend: the estimated single-model cost")
savings_estimated_turns: int = Field(
description="Turns covered by the current savings estimator; legacy estimates are excluded"
)
savings_estimated_actual_spend: float = Field(
description="Actual spend, including classifier cost, for covered turns only"
)
saved_spend: float | None = Field(description="Estimated savings for covered turns only, net of classifier cost")
baseline_spend: float | None = Field(
description="Estimated single-model cost; unavailable unless every turn is covered"
)
savings_estimated_baseline_spend: float | None = Field(
description="Estimated single-model cost for covered turns only"
)
baseline_model: str | None = Field(
description="The savings baseline most of this session's turns were priced against, recorded turn by "
description="The savings baseline most covered turns were priced against, recorded turn by "
"turn, so it still names the counterfactual after the router is reconfigured or removed. None when no "
"turn recorded one: rows from before the baseline was recorded, and adaptive and quality routers, "
"which derive no baseline and so report no savings"
)
baseline_models: Mapping[str, int] = Field(
description="Turns priced against each baseline model; more than one entry means the router's "
description="Covered turns priced against each baseline model; more than one entry means the router's "
"baseline changed mid-session and baseline_spend mixes both"
)

7
litellm/types/router.py
View file

@ -1036,6 +1036,13 @@ class TaggedPreRoutingStrategy(Generic[_PreRoutingStrategyT_co]):
strategy: _PreRoutingStrategyT_co
@dataclass(frozen=True, slots=True)
class BaselineRouteStamp:
router_name: str
baseline_model: str
baseline_deployment_id: str
@dataclass(frozen=True, slots=True)
class ConsumedRequestTagsStamp:
"""The model group a tagged router rewrote to, plus the request tags spent selecting it."""

5
litellm/types/utils.py
View file

@ -146,6 +146,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
supports_assistant_prefill: bool | None
supports_prompt_caching: bool | None
supports_prompt_cache_breakpoint: ReadOnly[bool | None]
supports_thinking_cache_preservation: ReadOnly[bool | None]
supports_computer_use: bool | None
supports_audio_input: bool | None
supports_embedding_image_input: bool | None
@ -3436,7 +3437,9 @@ class StandardLoggingPayload(ClassifierAudit):
stream: bool | None
response_cost: float
cost_breakdown: CostBreakdown | None # Detailed cost breakdown
autorouter_savings: ReadOnly[float | None] # None = not an auto-routed caller request; 0.0 is a real figure
autorouter_savings: ReadOnly[float | None]
autorouter_savings_estimate: ReadOnly[Mapping[str, JsonValue] | None]
autorouter_baseline_observation: ReadOnly[str | None]
response_cost_failure_debug_info: StandardLoggingModelCostFailureDebugInformation | None
status: StandardLoggingPayloadStatus
status_fields: StandardLoggingPayloadStatusFields

11
litellm/utils.py
View file

@ -1855,6 +1855,7 @@ def client(original_function):
# Type assertion: logging_obj is guaranteed to be non-None after function_setup
assert logging_obj is not None, "logging_obj should not be None after function_setup"
kwargs["litellm_logging_obj"] = logging_obj
modified_kwargs: Final = await async_pre_call_deployment_hook(kwargs, call_type)
if modified_kwargs is not None:
kwargs = modified_kwargs
@ -2817,6 +2818,14 @@ def supports_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None
)
def supports_thinking_cache_preservation(model: str, custom_llm_provider: str | None = None) -> bool:
return _supports_factory(
model=model,
custom_llm_provider=custom_llm_provider,
key="supports_thinking_cache_preservation",
)
def supports_computer_use(model: str, custom_llm_provider: str | None = None) -> bool:
"""
Check if the given model supports computer use and return a boolean value.
@ -5759,6 +5768,7 @@ def _get_model_info_helper(
supports_assistant_prefill=None,
supports_prompt_caching=None,
supports_prompt_cache_breakpoint=None,
supports_thinking_cache_preservation=None,
supports_computer_use=None,
supports_pdf_input=None,
)
@ -6031,6 +6041,7 @@ def _get_model_info_helper(
supports_assistant_prefill=_model_info.get("supports_assistant_prefill", None),
supports_prompt_caching=_model_info.get("supports_prompt_caching", None),
supports_prompt_cache_breakpoint=_model_info.get("supports_prompt_cache_breakpoint", None),
supports_thinking_cache_preservation=_model_info.get("supports_thinking_cache_preservation", None),
supports_audio_input=_model_info.get("supports_audio_input", None),
supports_audio_output=_model_info.get("supports_audio_output", None),
supports_pdf_input=_model_info.get("supports_pdf_input", None),

10
model_prices_and_context_window.json
View file

@ -13562,6 +13562,7 @@
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
@ -13600,6 +13601,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -13752,6 +13754,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -13782,6 +13785,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -13815,6 +13819,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -13853,6 +13858,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -13889,6 +13895,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -13928,6 +13935,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -14048,6 +14056,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,
@ -14090,6 +14099,7 @@
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_thinking_cache_preservation": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_native_structured_output": true,

3
model_prices_and_context_window.schema.json
View file

@ -794,6 +794,9 @@
"supports_system_messages": {
"type": "boolean"
},
"supports_thinking_cache_preservation": {
"type": "boolean"
},
"supports_tool_choice": {
"type": "boolean"
},

34
schema.prisma
View file

@ -1508,6 +1508,36 @@ model LiteLLM_AdaptiveRouterSession {
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
}
model LiteLLM_AutoRouterBaselineComparison {
scope String @id
api_key String
session_id String
router_name String
initial_equivalent Boolean
revision BigInt @default(0)
published_revision BigInt @default(0)
history String?
attempted_at DateTime?
retired Boolean @default(false)
updated_at DateTime @default(now())
@@index([api_key, session_id, router_name], map: "idx_autorouter_baseline_scope")
@@index([updated_at], map: "idx_autorouter_baseline_updated")
}
model LiteLLM_AutoRouterBaselineObservation {
request_id String @id
scope String
started_at Float
revision BigInt
data String
publication String?
conflicted Boolean @default(false)
@@index([scope, started_at, request_id], map: "idx_autorouter_baseline_event_order")
@@index([scope, revision, started_at], map: "idx_autorouter_baseline_event_revision")
}
model LiteLLM_AutoRouterSession {
api_key String
session_id String
@ -1534,6 +1564,10 @@ model LiteLLM_AutoRouterSession {
total_tokens BigInt @default(0)
spend Float @default(0)
saved_spend Float @default(0)
savings_estimated_turns Int @default(0)
savings_estimated_actual_spend Float @default(0)
savings_estimated_saved_spend Float @default(0)
savings_estimated_baseline_models Json @default("{}")
classifier_cost Float @default(0)
classifier_cost_recorded_turns Int @default(0)
tier_turns Json @default("{}")

34
tests/proxy_behavior/spend/test_autorouter_session_rollup.py
View file

@ -11,6 +11,7 @@ from datetime import datetime, timedelta, timezone
from typing import Final
import pytest
from prisma import Prisma
from litellm.proxy.db.autorouter_session_rollup import (
AUTOROUTER_BENCHMARKS_SQL,
@ -43,6 +44,7 @@ async def _turn(
classifier_cost: float = 0.0,
tier: "str | None" = None,
baseline: "str | None" = None,
estimated: bool = True,
) -> None:
touched: Final = 1 if (hit or ttl is not None or not covered) else 0
await db.execute_raw(
@ -63,6 +65,9 @@ async def _turn(
touched,
tier,
baseline,
int(estimated),
spend if estimated else 0.0,
saved if estimated else 0.0,
)
@ -208,6 +213,9 @@ async def test_subtotal_coverage_survives_legacy_and_rolling_writers(db, writers
assert row["saved_spend"] == pytest.approx(0.02 * len(writers))
assert row["classifier_cost"] == pytest.approx(0.004 * sum(writers))
assert row["classifier_cost_recorded_turns"] == sum(writers)
assert row["savings_estimated_turns"] == sum(writers)
assert row["savings_estimated_actual_spend"] == pytest.approx(0.01 * sum(writers))
assert row["savings_estimated_saved_spend"] == pytest.approx(0.02 * sum(writers))
groups: Final = await db.query_raw(
AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key
)
@ -217,6 +225,32 @@ async def test_subtotal_coverage_survives_legacy_and_rolling_writers(db, writers
assert groups[0]["turns"] == len(writers)
assert groups[0]["spend"] == row["spend"]
assert groups[0]["saved_spend"] == row["saved_spend"]
assert groups[0]["savings_estimated_turns"] == sum(writers)
assert groups[0]["savings_estimated_actual_spend"] == row["savings_estimated_actual_spend"]
assert groups[0]["savings_estimated_saved_spend"] == row["savings_estimated_saved_spend"]
async def test_unknown_and_legacy_turns_preserve_actual_spend_without_entering_the_estimated_cohort(db: Prisma) -> None:
key: Final = f"k-{uuid.uuid4()}"
await _turn(db, key, "A", T0, spend=0.25, saved=-0.05, baseline="opus")
await _turn(
db, key, "B", T0 + timedelta(seconds=1), spend=0.7, saved=0, baseline="sonnet", estimated=False
)
await _legacy_turn(db, key, T0 + timedelta(seconds=2))
row: Final = await _row(db, key)
assert row["saved_spend"] == pytest.approx(-0.03)
assert row["savings_estimated_baseline_models"] == {"opus": 1}
groups: Final = await db.query_raw(
AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key
)
assert len(groups) == 1
for actual in (row, groups[0]):
assert actual["turns"] == 3
assert actual["spend"] == pytest.approx(0.96)
assert actual["savings_estimated_turns"] == 1
assert actual["savings_estimated_actual_spend"] == pytest.approx(0.25)
assert actual["savings_estimated_saved_spend"] == pytest.approx(-0.05)
async def test_the_benchmarks_aggregate_reads_only_overlapping_sessions(db):

263
tests/proxy_behavior/spend/test_baseline_accounting.py Normal file
View file

@ -0,0 +1,263 @@
import asyncio
import json
import uuid
from collections.abc import AsyncIterator, Callable
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from typing import Final
import pytest
from prisma import Prisma
import litellm
from litellm.llms.anthropic.prompt_cache_prediction import CountedBreakpoint, CountedPromptCachePlan
from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
from litellm.proxy.db.baseline_accounting import (
BaselineAccountingRecord,
BaselineAccountingStore,
DailyBaselineAttribution,
DailyBaselineTarget,
)
from litellm.proxy.db.create_views import SupportsRawQueries
from litellm.proxy.spend_tracking.baseline_accounting import BaselineObservation
from litellm.proxy.spend_tracking.savings import BaselineCostSnapshot
from litellm.types.utils import Usage
pytestmark = pytest.mark.asyncio(loop_scope="session")
@asynccontextmanager
async def _transaction(db: Prisma, *, before_commit: bool = False, after_commit: bool = False) -> AsyncIterator[SupportsRawQueries]:
async with db.tx() as tx:
yield tx
if before_commit:
raise RuntimeError("injected pre-commit interruption")
if after_commit:
raise RuntimeError("injected lost commit acknowledgement")
def _store(db: Prisma, **faults: bool) -> BaselineAccountingStore:
def transaction():
return _transaction(db, **faults)
return BaselineAccountingStore(transaction)
@pytest.fixture
def record() -> Callable[..., BaselineAccountingRecord]:
run: Final = uuid.uuid4().hex
marker: Final = CountedBreakpoint("prefix", 3600, 6000, ("prefix",), "content", ("content",))
usage: Final = Usage(
prompt_tokens=6200, completion_tokens=30, total_tokens=6230,
cache_creation_input_tokens=6000, cache_read_input_tokens=0,
prompt_tokens_details={
"text_tokens": 200, "cached_tokens": 0, "cache_creation_tokens": 6000,
"cache_creation_token_details": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 6000},
},
)
def create(label: str = "first", started: float = 10000.0, identical: bool = True) -> BaselineAccountingRecord:
return BaselineAccountingRecord(
scope="autorouter-baseline:v3:" + run * 2, api_key=run, session_id=run,
router_name="test-router", baseline_model="anthropic/claude-opus-5",
observation=BaselineObservation(
request_id=run + label, started_at=started, available_at=started + 0.1,
outcome="complete", baseline_equivalent=identical, usage=usage,
plan=CountedPromptCachePlan(6200, (marker,)), minimum_cache_tokens=4096,
),
pricing=BaselineCostSnapshot(
model="claude-opus-5", provider="anthropic",
prices=litellm.get_model_info("claude-opus-5", custom_llm_provider="anthropic"),
actual_spend=0.17, actual_token_cost=0.17,
),
turn=AutoRouterTurnTransaction(
api_key=run, session_id=run, router_name="test-router", router_type="heuristic",
model="claude-opus-5", turn_at=datetime.fromtimestamp(started, timezone.utc),
total_tokens=6230, spend=0.17, saved_spend=0.0, classifier_cost=0.0,
covered=True, cache_hit=False, cache_ttl_seconds=3600, cache_touched=True,
baseline_model="anthropic/claude-opus-5",
),
daily=DailyBaselineAttribution(
date="2026-09-15", api_key=run, model="claude-opus-5", custom_llm_provider="anthropic",
targets=tuple(DailyBaselineTarget(entity=entity, entity_id=run) for entity in ("user", "team", "org", "end_user", "agent", "tag")),
),
)
return create
async def _log(db: Prisma, record: BaselineAccountingRecord) -> None:
await db.execute_raw(
'INSERT INTO "LiteLLM_SpendLogs" (request_id,call_type,api_key,spend,"startTime","endTime") '
"VALUES ($1, 'anthropic_messages', $2, 0.17, to_timestamp($3::float8), to_timestamp($3::float8))",
record.observation.request_id, record.api_key, record.observation.started_at,
)
async def _session(db: Prisma, record: BaselineAccountingRecord):
rows: Final = await db.query_raw('SELECT * FROM "LiteLLM_AutoRouterSession" WHERE api_key=$1', record.api_key)
return rows[0]
async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
store: Final = _store(db)
late: Final = record("late", 10001.0)
early: Final = record("early", identical=False)
await _log(db, late)
assert await store.append(late) == "recorded"
assert await store.project(late.scope) == "published"
before: Final = await _session(db, late)
assert before["savings_estimated_actual_spend"] == before["spend"] == 0.17
assert before["saved_spend"] == 0.0
await _log(db, early)
assert await store.append(early) == "recorded"
pending: Final = await _session(db, late)
assert pending["spend"] == 0.34 and pending["savings_estimated_turns"] == 0
assert pending["saved_spend"] == pending["savings_estimated_actual_spend"] == 0.0
waiting: Final = await db.query_raw('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=$1', late.observation.request_id)
assert waiting[0]["metadata"]["autorouter_savings"] is None
assert waiting[0]["metadata"]["autorouter_savings_estimate"]["reason"] == "pending_projection"
assert await store.project(early.scope) == "published"
after: Final = await _session(db, late)
assert after["spend"] == 0.34 and after["turns"] == 2
assert after["savings_estimated_actual_spend"] == 0.17 and after["savings_estimated_turns"] == 1
logs: Final = await db.query_raw('SELECT spend, metadata FROM "LiteLLM_SpendLogs" WHERE request_id=$1', late.observation.request_id)
assert logs[0]["spend"] == 0.17
assert logs[0]["metadata"]["autorouter_savings_estimate"]["provenance"] == "modeled"
assert after["saved_spend"] == pytest.approx(logs[0]["metadata"]["autorouter_savings"])
for table in ("DailyUserSpend", "DailyTeamSpend", "DailyOrganizationSpend", "DailyEndUserSpend", "DailyAgentSpend", "DailyTagSpend"):
rows: Final = await db.query_raw(f'SELECT spend,api_requests,autorouter_savings_spend FROM "LiteLLM_{table}" WHERE api_key=$1', late.api_key)
assert rows[0]["spend"] == rows[0]["api_requests"] == 0
assert rows[0]["autorouter_savings_spend"] == pytest.approx(after["saved_spend"])
async def test_commit_ack_loss_and_concurrent_duplicate_delivery_are_idempotent(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
event: Final = record()
await _log(db, event)
assert await _store(db, after_commit=True).append(event) == "unavailable"
store: Final = _store(db)
assert set(await asyncio.gather(*(store.append(event) for _ in range(4)))) == {"recorded"}
assert await store.project(event.scope) == "published"
assert await store.project(event.scope) == "unchanged"
session: Final = await _session(db, event)
assert session["turns"] == session["savings_estimated_turns"] == 1
assert session["spend"] == session["savings_estimated_actual_spend"] == 0.17
async def test_publication_rollback_keeps_dirty_revision_for_retry(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
event: Final = record()
await _log(db, event)
store: Final = _store(db)
assert await store.append(event) == "recorded"
assert await _store(db, before_commit=True).project(event.scope) == "unavailable"
session: Final = await _session(db, event)
assert session["spend"] == 0.17 and session["savings_estimated_turns"] == 0
revisions: Final = await db.query_raw('SELECT revision,published_revision FROM "LiteLLM_AutoRouterBaselineComparison" WHERE scope=$1', event.scope)
assert revisions[0]["revision"] > revisions[0]["published_revision"]
assert await store.project(event.scope) == "published"
assert (await _session(db, event))["savings_estimated_turns"] == 1
async def test_conflicting_duplicate_cannot_restore_an_observed_estimate(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
event: Final = record()
await _log(db, event)
store: Final = _store(db)
assert await store.append(event) == "recorded"
assert await store.project(event.scope) == "published"
conflict: Final = event.model_copy(update={"observation": event.observation.model_copy(update={"baseline_equivalent": False, "started_at": 20000.0, "available_at": 20001.0})})
assert await store.append(conflict) == "recorded"
assert (await _session(db, event))["savings_estimated_turns"] == 0
assert await store.append(event) == "recorded"
assert await store.project(event.scope) == "published"
session: Final = await _session(db, event)
assert session["turns"] == 1 and session["savings_estimated_turns"] == 0
rows: Final = await db.query_raw('SELECT publication FROM "LiteLLM_AutoRouterBaselineObservation" WHERE request_id=$1', event.observation.request_id)
assert json.loads(rows[0]["publication"])["reason"] == "conflicting_observation"
async def test_retired_history_never_recreates_an_initial_zero(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
original: Final = record()
await _log(db, original)
store: Final = _store(db)
assert await store.append(original) == "recorded"
assert await store.project(original.scope) == "published"
await db.execute_raw('UPDATE "LiteLLM_AutoRouterBaselineComparison" SET updated_at=to_timestamp(0) WHERE scope=$1', original.scope)
await store.retire_before(datetime(2000, 1, 1, tzinfo=timezone.utc), 1000, 1000)
next_turn: Final = record("after-retention", 20000.0)
await _log(db, next_turn)
assert await store.append(next_turn) == "retired"
assert await store.project(original.scope) == "unchanged"
after: Final = await _session(db, original)
assert after["turns"] == 2 and after["spend"] == 0.34
assert after["savings_estimated_turns"] == 1 and after["savings_estimated_actual_spend"] == 0.17
async def test_native_observation_enters_spend_pipeline_once_with_shared_daily_attribution(
db: Prisma, record: Callable[..., BaselineAccountingRecord], monkeypatch: pytest.MonkeyPatch,
) -> None:
import os
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
from litellm.proxy.hooks.autorouter_baseline_cache import CapturedBaselineObservation
from litellm.proxy.utils import PrismaClient, ProxyLogging
event: Final = record("routed", identical=False)
capture: Final = CapturedBaselineObservation(
scope=event.scope, api_key=event.api_key, session_id=event.session_id,
router_name=event.router_name, baseline_model=event.baseline_model,
model=event.pricing.model, prices=event.pricing.prices, observation=event.observation,
)
metadata: Final = {
"routing_decision": {"router_model_name": event.router_name, "savings_baseline_model": event.baseline_model},
"usage_object": event.observation.usage.model_dump(),
"cost_breakdown": {"input_cost": 0.16, "output_cost": 0.01},
"autorouter_savings": None, "autorouter_savings_estimate": {"version": 3, "status": "unknown", "reason": "pending_projection"},
"autorouter_baseline_observation": capture.model_dump_json(),
}
payload: Final = {
"request_id": event.observation.request_id, "api_key": event.api_key, "session_id": event.session_id,
"startTime": datetime.fromtimestamp(event.observation.started_at, timezone.utc).isoformat(),
"endTime": datetime.fromtimestamp(event.observation.available_at, timezone.utc).isoformat(),
"spend": 0.17, "prompt_tokens": 6200, "completion_tokens": 30, "model": event.pricing.model,
"model_group": event.router_name, "model_id": "baseline", "custom_llm_provider": "anthropic",
"call_type": "anthropic_messages", "status": "success", "metadata": json.dumps(metadata),
"user": None, "team_id": "", "organization_id": "org", "agent_id": None,
"end_user": "", "request_tags": '["tag","tag"]',
}
monkeypatch.delenv("DATABASE_URL_READ_REPLICA", raising=False)
client: Final = PrismaClient(os.environ["DATABASE_URL"], ProxyLogging(UserApiKeyCache()))
writer: Final = DBSpendUpdateWriter()
try:
await client.db.connect()
await _log(db, event)
await writer._enqueue_autorouter_turn_transaction(payload, client)
assert len(client.baseline_accounting_transactions) == 1
queued: Final = client.baseline_accounting_transactions[0]
assert queued.daily is not None
assert [(target.entity, target.entity_id) for target in queued.daily.targets] == [
("user", None), ("team", ""), ("org", "org"), ("tag", "tag"),
]
await writer.add_spend_log_transaction_to_daily_tag_transaction(payload, client)
actual_tags: Final = await writer.daily_tag_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
assert len(actual_tags) == 1
assert next(iter(actual_tags.values()))["spend"] == 0.17
durable: Final = BaselineAccountingStore.for_client(client)
anchor: Final = record("anchor", 9999.0)
await _log(db, anchor)
assert await durable.append(anchor) == "recorded"
assert await durable.append(queued) == "recorded"
assert await durable.append(queued) == "recorded"
assert await durable.project(queued.scope) == "published"
session: Final = await _session(db, queued)
assert session["turns"] == session["savings_estimated_turns"] == 2
assert session["spend"] == session["savings_estimated_actual_spend"] == 0.34
tag_rows: Final = await db.query_raw(
'SELECT spend, api_requests, autorouter_savings_spend FROM "LiteLLM_DailyTagSpend" WHERE api_key=$1 AND tag=$2',
queued.api_key, "tag",
)
assert session["saved_spend"] < 0
assert len(tag_rows) == 1
assert tag_rows[0]["autorouter_savings_spend"] == pytest.approx(session["saved_spend"])
assert tag_rows[0]["spend"] == tag_rows[0]["api_requests"] == 0
finally:
await client.db.disconnect()

103
tests/proxy_migration_tests/test_autorouter_baseline_state.py Normal file
View file

@ -0,0 +1,103 @@
"""Idempotent journal migration and primary transactional ownership."""
import asyncio
import os
import time
from collections.abc import AsyncGenerator, Iterator
from contextlib import asynccontextmanager
from datetime import timedelta
from pathlib import Path
from typing import Final
from uuid import uuid4
import psycopg
import pytest
from psycopg import sql
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.db.baseline_accounting import BaselineAccountingStore
from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
from litellm.proxy.utils import PrismaClient, ProxyLogging
_MIGRATION: Final = Path(__file__).parents[2] / (
"litellm-proxy-extras/litellm_proxy_extras/migrations/20260915010000_add_autorouter_baseline_state/migration.sql"
)
@pytest.fixture
def database() -> Iterator[tuple[str, psycopg.Connection[tuple[object, ...]]]]:
base: Final = os.environ["DATABASE_URL"].split("?")[0]
schema: Final = f"baseline_{uuid4().hex}"
with psycopg.connect(base, autocommit=True) as connection:
connection.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema)))
connection.execute(sql.SQL("SET search_path TO {}").format(sql.Identifier(schema)))
try:
connection.execute(_MIGRATION.read_bytes())
connection.execute(_MIGRATION.read_bytes())
yield f"{base}?schema={schema}", connection
finally:
connection.execute(sql.SQL("DROP SCHEMA {} CASCADE").format(sql.Identifier(schema)))
@asynccontextmanager
async def _client(env: pytest.MonkeyPatch, url: str, replica: str | None = None) -> AsyncGenerator[PrismaClient]:
with env.context() as context:
context.setenv("DATABASE_URL", url)
context.delenv("DATABASE_URL_READ_REPLICA", raising=False)
if replica is not None:
context.setenv("DATABASE_URL_READ_REPLICA", replica)
client: Final = PrismaClient(url, ProxyLogging(UserApiKeyCache()))
try:
await client.db.connect(timeout=timedelta(seconds=1))
yield client
finally:
await client.db.disconnect()
@pytest.mark.asyncio
async def test_migration_and_projector_use_the_primary_across_clients(
database: tuple[str, psycopg.Connection[tuple[object, ...]]], monkeypatch: pytest.MonkeyPatch,
) -> None:
url, connection = database
connection.execute('CREATE TABLE "LiteLLM_SpendLogs" (request_id TEXT PRIMARY KEY)')
connection.execute('INSERT INTO "LiteLLM_AutoRouterBaselineComparison" '
'(scope,api_key,session_id,router_name,initial_equivalent,revision) '
"VALUES ('test','key','session','router',TRUE,1)")
async with _client(monkeypatch, url, url.split("?")[0]) as first:
assert await BaselineAccountingStore.for_client(first).project("test") == "published"
async with _client(monkeypatch, url) as restarted:
assert await BaselineAccountingStore.for_client(restarted).project("test") == "unchanged"
assert connection.execute('SELECT revision=published_revision FROM "LiteLLM_AutoRouterBaselineComparison"').fetchone() == (True,)
@pytest.mark.asyncio
async def test_primary_outage_and_missing_table_are_unavailable(
database: tuple[str, psycopg.Connection[tuple[object, ...]]], monkeypatch: pytest.MonkeyPatch,
) -> None:
url, connection = database
async with _client(monkeypatch, "postgresql://unused:unused@127.0.0.1:1/unreachable", url) as degraded:
assert isinstance(degraded.db, RoutingPrismaWrapper) and degraded.db.writer_unavailable
assert await BaselineAccountingStore.for_client(degraded).project("scope") == "unavailable"
connection.execute('DROP TABLE "LiteLLM_AutoRouterBaselineComparison"')
async with _client(monkeypatch, url) as missing:
assert await BaselineAccountingStore.for_client(missing).project("scope") == "unavailable"
@pytest.mark.asyncio
async def test_locked_projection_is_bounded_and_cancellation_propagates(
database: tuple[str, psycopg.Connection[tuple[object, ...]]], monkeypatch: pytest.MonkeyPatch,
) -> None:
url, connection = database
async with _client(monkeypatch, url) as client:
store: Final = BaselineAccountingStore.for_client(client)
with connection.transaction():
connection.execute('LOCK TABLE "LiteLLM_AutoRouterBaselineComparison" IN ACCESS EXCLUSIVE MODE')
started: Final = time.monotonic()
assert await store.project("scope") == "unavailable"
assert time.monotonic() - started < 2
pending: Final = asyncio.create_task(store.project("scope"))
await asyncio.sleep(0.01)
pending.cancel()
with pytest.raises(asyncio.CancelledError):
await pending
assert await store.project("scope") == "unchanged"

9
tests/proxy_unit_tests/test_update_spend.py
View file

@ -37,10 +37,15 @@ class MockPrismaClient:
self.daily_user_spend_transactions = {}
self.tool_usage_transactions = []
self.autorouter_turn_transactions = []
self.baseline_accounting_transactions = []
self.baseline_accounting_lock = asyncio.Lock()
self.spend_log_flush_requested = None
self.db.tx = MagicMock()
self.db.tx.return_value.__aenter__ = AsyncMock(return_value=self.db)
self.db.tx.return_value.__aexit__ = AsyncMock(return_value=None)
self.db.query_raw.return_value = []
# Add locks for the transaction queues (matches real PrismaClient)
import asyncio
self._spend_log_transactions_lock = asyncio.Lock()
self._tool_usage_transactions_lock = asyncio.Lock()
self._autorouter_turn_transactions_lock = asyncio.Lock()

24
tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -124,22 +124,32 @@ def test_calculate_usage_prefers_served_speed_from_response_usage():
assert no_response_speed.speed == "fast"
def test_streaming_iterator_persists_served_speed_across_usage_chunks():
@pytest.mark.parametrize("input_update, expected_fresh", [({}, 1000), ({"input_tokens": 0}, 0), ({"input_tokens": 2000}, 2000)])
def test_streaming_iterator_persists_cumulative_usage_across_partial_chunks(input_update, expected_fresh):
"""
Only ``message_start`` usage carries the served speed; the final
``message_delta`` usage does not. The iterator must remember the served
value so the last usage chunk, which wins in the stream chunk builder, does
not fall back to the requested speed.
Omitted input/cache/pricing fields retain their last cumulative values;
explicit input updates, including zero, replace them.
"""
from litellm.llms.anthropic.chat.handler import ModelResponseIterator
iterator = ModelResponseIterator(None, sync_stream=True, speed="fast")
start_usage = iterator._handle_usage({"input_tokens": 12, "output_tokens": 1, "speed": "standard"})
delta_usage = iterator._handle_usage({"output_tokens": 5})
start_usage = iterator._handle_usage({
"input_tokens": 1000, "output_tokens": 1, "speed": "standard", "inference_geo": "us",
"cache_creation_input_tokens": 3000, "cache_read_input_tokens": 2000,
"cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3000},
})
delta_usage = iterator._handle_usage({"output_tokens": 5, **input_update})
assert start_usage.speed == "standard"
assert delta_usage.speed == "standard"
assert delta_usage.inference_geo == "us"
assert delta_usage.prompt_tokens == expected_fresh + 5000
assert delta_usage.completion_tokens == 5
details = delta_usage.prompt_tokens_details
assert (details.text_tokens, details.cached_tokens, details.cache_creation_tokens) == (expected_fresh, 2000, 3000)
assert details.cache_creation_token_details.ephemeral_1h_input_tokens == 3000
assert start_usage.prompt_tokens_details.text_tokens == 1000
def test_calculate_usage_aggregates_cache_creation_split_across_iterations():

235
tests/test_litellm/llms/anthropic/test_anthropic_prompt_cache_prediction.py
View file

@ -15,11 +15,17 @@ from litellm.caching.llm_caching_handler import LLMClientCache
from litellm.llms.anthropic.count_tokens import handler as count_handler
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import DEFAULT_ANTHROPIC_API_VERSION
from litellm.llms.anthropic.prompt_cache_prediction import (
CountedPromptCachePlan,
NativePredictionTarget,
PromptCachePlan,
UnsupportedCachePlan,
cache_scope,
count_cache_plan,
count_prompt_tokens,
parse_cache_plan,
parse_observed_cache,
parse_prompt,
resolve_baseline_prediction_target,
resolve_prediction_target,
supported_prediction_headers,
)
@ -207,3 +213,232 @@ async def test_named_credential_is_explicitly_unsupported_before_count(
assert arm.cache_state == "unknown"
assert arm.reason == "unsupported_deployment_configuration"
assert arm.estimate is None and arm.cold is None and arm.warm is None
def _cache_plan(body: Mapping[str, JsonValue]) -> PromptCachePlan:
plan: Final = parse_cache_plan(body)
assert isinstance(plan, PromptCachePlan)
return plan
def _text(text: str, ttl: str | None = None) -> dict[str, JsonValue]:
return {"type": "text", "text": text,
**({"cache_control": {"type": "ephemeral", "ttl": ttl}} if ttl else {})}
def _prompt(*blocks: dict[str, JsonValue], role: str = "user", **options: JsonValue) -> dict[str, JsonValue]:
return {**options, "messages": [{"role": role, "content": list(blocks)}]}
@pytest.mark.parametrize("text, supported", [("", False), (" \t", False), ("Context", True)])
def test_public_predictor_preserves_string_message_policy(text: str, supported: bool) -> None:
body: Final = _body()
messages: Final = body["messages"]
assert isinstance(messages, list)
request: Final[dict[str, JsonValue]] = {**body, "messages": [{"role": "user", "content": text}, *messages]}
assert (parse_prompt(request) is not None) is supported
def test_cache_plan_preserves_hierarchical_prefixes_and_public_policy() -> None:
body: Final = _prompt(
_text("First turn", "5m"), system=[_text("Stable instructions", "1h")],
tools=[{"name": "lookup", "input_schema": {"type": "object"},
"cache_control": {"type": "ephemeral", "ttl": "1h"}}],
)
plan: Final = _cache_plan(body)
changed: Final = _cache_plan({**body, "system": [_text("Changed instructions", "1h")]})
assert tuple(marker.ttl_seconds for marker in plan.breakpoints) == (3600, 3600, 300)
assert plan.breakpoints[0].fingerprint == changed.breakpoints[0].fingerprint
assert all(left.fingerprint != right.fingerprint for left, right
in zip(plan.breakpoints[1:], changed.breakpoints[1:]))
assert plan.breakpoints[0].prefix_body == {
"tools": [{"name": "lookup", "input_schema": {"type": "object"}}],
"messages": [],
}
assert parse_prompt(body) is None
@pytest.mark.parametrize("kind, added, matches", [
("text", 19, True), ("text", 20, False), ("tool_use", 30, True),
("tool_result", 30, True),
])
def test_cache_plan_lookback_counts_native_positions(
kind: str, added: int, matches: bool,
) -> None:
previous: Final = _cache_plan(_body())
appended: Final[list[dict[str, JsonValue]]] = [
{"type": "tool_use", "id": f"tool_{index}", "name": "lookup", "input": {}}
if kind == "tool_use" else
{"type": "tool_result", "tool_use_id": f"tool_{index}", "content": "done"}
if kind == "tool_result" else
{"type": "text", "text": f"Added {index}"}
for index in range(added)
]
current: Final = _cache_plan({**_body(), **_prompt(
_text("A cacheable prefix"), *appended[:-1],
{**appended[-1], "cache_control": {"type": "ephemeral"}},
)})
assert (previous.breakpoints[0].fingerprint
in current.breakpoints[0].lookback_fingerprints) is matches
@pytest.mark.parametrize("change, same_prefix, same_content", [
("tool_order", False, False), ("effort", False, False),
("standard_speed", True, True), ("ttl", False, True),
])
def test_cache_plan_identity_respects_settings_and_preserves_content(
change: str, same_prefix: bool, same_content: bool,
) -> None:
tool_input: Final[dict[str, JsonValue]] = {"a": 1, "b": 2, "cache_control": {"ttl": "user-data"}}
block: Final[dict[str, JsonValue]] = {
"type": "tool_use", "id": "tool_1", "name": "lookup", "input": tool_input,
"cache_control": {"type": "ephemeral", "ttl": "5m"},
}
changed_block: Final = (
{**block, "input": dict(reversed(tool_input.items()))} if change == "tool_order" else
{**block, "cache_control": {"type": "ephemeral", "ttl": "1h"}} if change == "ttl" else block
)
before: Final = _cache_plan(_prompt(block, role="assistant", output_config={"effort": "low"})).breakpoints[0]
after: Final = _cache_plan(_prompt(
changed_block, role="assistant", output_config={"effort": "high" if change == "effort" else "low"},
**({"speed": "standard"} if change == "standard_speed" else {}),
)).breakpoints[0]
assert (before.fingerprint == after.fingerprint) is same_prefix
assert (before.fingerprint in after.lookback_fingerprints) is same_prefix
assert (before.content_fingerprint == after.content_fingerprint) is same_content
assert (before.content_fingerprint in after.lookback_content_fingerprints) is same_content
assert "user-data" in json.dumps(dict(before.prefix_body))
assert not supported_prediction_headers({"anthropic-beta": "fast-mode-2026-02-01"})
def test_cache_plan_automatic_cache_and_thinking_use_last_cacheable_block() -> None:
body: Final = _prompt(
_text("A stable answer"), {"type": "thinking", "thinking": "Thinking", "signature": "signature"},
role="assistant", thinking={"type": "adaptive"}, cache_control={"type": "ephemeral", "ttl": "1h"},
)
plan: Final = _cache_plan(body)
assert len(plan.breakpoints) == 1
assert plan.breakpoints[0].ttl_seconds == 3600
assert plan.breakpoints[0].prefix_body == {
"thinking": {"type": "adaptive"},
"messages": [{"role": "assistant", "content": [
{"type": "text", "text": "A stable answer"},
]}],
}
assert parse_prompt(body) is None
@pytest.mark.parametrize("body, reason", [
(_prompt({"type": "image"}), "unsupported_prompt_shape"),
({**_body(), "unknown_native_setting": True}, "unsupported_prompt_shape"),
({**_body(), "cache_control": {"type": "ephemeral", "ttl": "1h"}},
"conflicting_cache_ttl"),
(_prompt(_text("five", "5m"), _text("hour", "1h")), "invalid_cache_ttl_order"),
])
def test_cache_plan_unsupported_is_explicit(
body: Mapping[str, JsonValue], reason: str,
) -> None:
result: Final = parse_cache_plan(body)
assert isinstance(result, UnsupportedCachePlan)
assert result.reason == reason
@pytest.mark.asyncio
@pytest.mark.parametrize("model, counts, reason", [
(None, (100, 150, 200), None),
(None, (100, 201, 200), "inconsistent_prefix_token_count"),
(None, (151, 150, 200), "inconsistent_prefix_token_count"),
(None, (None, 150, 200), "token_count_unavailable"),
("claude-opus-5", (100, 150, 200), None),
("claude-sonnet-5", (100, 150, 200), None),
("declared-cache-model", (100, 150, 200), None),
("unknown-cache-model", (100, 150, 200), "unsupported_thinking_cache_semantics"),
("claude-haiku-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"),
("claude-sonnet-4-5", (100, 150, 200), "unsupported_thinking_cache_semantics"),
])
async def test_cache_plan_count_conserves_total_and_rejects_unknown(
model: str | None, counts: tuple[int | None, int | None, int], reason: str | None,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setitem(litellm.model_cost, "declared-cache-model", {
"litellm_provider": "anthropic", "mode": "chat", "supports_thinking_cache_preservation": True,
})
plan: Final = _cache_plan(_prompt(
{"type": "thinking", "thinking": "Retained thought", "signature": "signature"}
if model else _text("first", "5m"),
_text("second", "5m"), _text("uncached"), role="assistant" if model else "user",
))
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
assert reason != "unsupported_thinking_cache_semantics", "Unverified thinking retention must skip counting"
if body is plan.full_body:
return counts[2]
return counts[0] if body is plan.breakpoints[0].prefix_body else counts[1]
result: Final = await count_cache_plan(model or _MODEL, _KEY, plan, count)
if reason is not None:
assert isinstance(result, UnsupportedCachePlan)
assert result.reason == reason
else:
assert isinstance(result, CountedPromptCachePlan)
assert result.total_tokens == 200
assert tuple(marker.prefix_tokens for marker in result.breakpoints) == ((100,) if model else (100, 150))
@pytest.mark.asyncio
@pytest.mark.parametrize("section", ["system", "tools"])
@pytest.mark.parametrize("rejects_prefix", (False, True))
async def test_native_count_preserves_settings_and_requires_every_prefix(
section: str, rejects_prefix: bool, monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
params: Final = LiteLLM_Params(
model=f"anthropic/{_MODEL}", api_key=_KEY,
api_base="https://gateway.example/v1/messages",
)
target: Final = resolve_baseline_prediction_target(params)
assert isinstance(target, NativePredictionTarget)
assert target.api_base == params.api_base
assert not isinstance(resolve_prediction_target(params), NativePredictionTarget)
body: Final = _body()
marker: Final[dict[str, JsonValue]] = {"type": "ephemeral", "ttl": "1h"}
body[section] = ([_text("A cached system", "1h")] if section == "system" else [{
"name": "lookup", "input_schema": {"type": "object"}, "cache_control": marker,
}])
plan: Final = _cache_plan({**body, **_prompt(
_text("A later prefix", "5m"), _text("An uncached suffix"),
thinking={"type": "adaptive"}, tool_choice={"type": "auto"}, output_config={"effort": "high"},
)})
assert len(plan.breakpoints) == 2
assert plan.breakpoints[0].prefix_body["messages"] == []
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
return await count_prompt_tokens(
model, api_key, {**body, "max_tokens": 100}, api_base=target.api_base,
)
with respx.mock(assert_all_called=False) as upstream:
endpoint: Final = "https://gateway.example/v1/messages/count_tokens"
routes: Final = tuple(
upstream.post(endpoint, json={**body, "model": _MODEL}).respond(
400 if rejects_prefix and index == 1 else 200,
json={"detail": {"error": "messages parameter is required"}}
if rejects_prefix and index == 1 else {"input_tokens": tokens},
)
for index, (body, tokens) in enumerate((
(plan.full_body, 6000), (plan.breakpoints[0].prefix_body, 5000),
(plan.breakpoints[1].prefix_body, 5800),
))
)
unexpected: Final = upstream.post(endpoint).respond(200, json={"input_tokens": 1})
result: Final = await count_cache_plan(target.model, target.api_key, plan, count)
if rejects_prefix:
assert result == UnsupportedCachePlan("token_count_unavailable")
else:
assert isinstance(result, CountedPromptCachePlan)
assert result.total_tokens == 6000
assert tuple(marker.prefix_tokens for marker in result.breakpoints) == (5000, 5800)
assert tuple(route.call_count for route in routes) == (1, 1, 1)
assert unexpected.call_count == 0

8
tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
View file

@ -1551,7 +1551,7 @@ async def test_anthropic_post_uses_prebuilt_body_without_redumping():
provider_config = Mock()
provider_config.max_retry_on_anthropic_messages_http_error = 2
logging_obj = Mock()
logging_obj: Final = Mock(baseline_cache_context=None)
logging_obj.model_call_details = {}
out = await handler._async_post_anthropic_messages_with_http_error_retry(
@ -1591,7 +1591,7 @@ async def test_anthropic_post_falls_back_to_json_dumps_when_unsigned_none():
provider_config = Mock()
provider_config.max_retry_on_anthropic_messages_http_error = 1
logging_obj = Mock()
logging_obj: Final = Mock(baseline_cache_context=None)
logging_obj.model_call_details = {}
await handler._async_post_anthropic_messages_with_http_error_retry(
@ -1639,7 +1639,7 @@ async def test_anthropic_post_retry_reserializes_mutated_body():
# Re-sign returns no signed body (native anthropic path) -> must re-dump.
provider_config.sign_request = Mock(return_value=({}, None))
logging_obj = Mock()
logging_obj: Final = Mock(baseline_cache_context=None)
logging_obj.model_call_details = {}
await handler._async_post_anthropic_messages_with_http_error_retry(
@ -2487,7 +2487,7 @@ async def test_anthropic_invalid_thinking_signature_retry_resigns_bedrock_reques
posts.append({"headers": dict(headers), "data": data})
return invalid_signature_response if len(posts) == 1 else ok_response
logging_obj = Mock()
logging_obj: Final = Mock(baseline_cache_context=None)
logging_obj.model_call_details = {}
response = await handler._async_post_anthropic_messages_with_http_error_retry(

8
tests/test_litellm/models/test_models.py
View file

@ -593,7 +593,7 @@ class TestManagedTables:
class TestAutoRouterSession:
@staticmethod
def _row(baseline_models: dict) -> LiteLLM_AutoRouterSession:
def _row(estimated_baseline_models: dict[str, int]) -> LiteLLM_AutoRouterSession:
return LiteLLM_AutoRouterSession(
api_key="k",
session_id="s",
@ -607,7 +607,9 @@ class TestAutoRouterSession:
saved_spend=0.24,
classifier_cost=0.0,
tier_turns={},
baseline_models=baseline_models,
baseline_models={"legacy-baseline": 100},
savings_estimated_turns=sum(estimated_baseline_models.values()),
savings_estimated_baseline_models=estimated_baseline_models,
)
def test_the_baseline_label_is_the_one_most_turns_were_priced_against(self):
@ -619,5 +621,5 @@ class TestAutoRouterSession:
assert self._row({"b-model": 1, "a-model": 1}).baseline_model == "b-model"
assert self._row({"a-model": 1, "b-model": 1}).baseline_model == "b-model"
def test_a_row_whose_turns_recorded_no_baseline_has_no_label(self):
def test_a_row_without_current_estimates_has_no_baseline_label(self) -> None:
assert self._row({}).baseline_model is None

40
tests/test_litellm/proxy/client/cli/test_statusline_script.py
View file

@ -325,9 +325,12 @@ class TestRender:
use_color=False,
)
def test_a_session_that_cost_more_than_its_baseline_reads_as_a_plus(self, config_dir):
dearer = RECORDED._replace(spend=0.50, baseline_spend=0.40)
assert "+25% vs Claude Opus 5" in render("m", dearer, config_dir, use_color=False)
@pytest.mark.parametrize("spend,delta", ((0.50, "+25%"), (0.40, "0%"), (0.4001, "0%"), (0.3999, "0%"), (0.30, "-25%")))
def test_rounded_cost_delta_uses_a_sign_only_for_nonzero_percentages(
self, config_dir: Path, spend: float, delta: str,
) -> None:
session: Final = RECORDED._replace(spend=spend, baseline_spend=0.40)
assert render("m", session, config_dir, use_color=False).splitlines()[0] == f"Routed to: m {delta} vs Claude Opus 5"
def test_without_a_baseline_only_the_routed_line_shows(self, config_dir):
assert render("m", RECORDED._replace(baseline_model=None), config_dir, False) == "Routed to: m"
@ -339,6 +342,37 @@ class TestRender:
class TestClaudeCodeMode:
@pytest.mark.parametrize("estimated_turns", (0, 1))
def test_current_estimates_keep_the_routed_model_and_compare_only_covered_turns(
self, tmp_path: Path, transcript: Path, config_dir: Path, estimated_turns: int
) -> None:
session: Final = statusline_script._session_from_payload(
{
**RECORDED._asdict(),
"spend": 10.0,
"baseline_spend": None,
"savings_estimated_baseline_spend": 1.5 if estimated_turns else None,
"turns": 3,
"savings_estimated_turns": estimated_turns,
"savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0,
}
)
assert session is not None
def fetch(credentials: Credentials, session_id: str) -> Fetched:
return Fetched(session, True)
first: Final = _run(_payload(transcript), _env(tmp_path, config_dir), fetch)
assert first == _run(_payload(transcript), _env(tmp_path, config_dir), fetch)
assert first.startswith("Routed to: claude-sonnet-5")
if estimated_turns:
assert "+33% vs Claude Opus 5 · 1 of 3 turns estimated" in first
assert "$2.00" in first and "$1.50" in first
assert "$10.00" not in first and "+567%" not in first
else:
assert "Savings unavailable" in first
assert "%" not in first and "$" not in first
@pytest.mark.parametrize("transcript_model", ("claude-auto", "anthropic/claude-opus-5"))
def test_the_session_names_the_routed_model_even_when_the_transcript_differs(
self, tmp_path: Path, config_dir: Path, transcript_model: str

26
tests/test_litellm/proxy/db/test_autorouter_session_rollup.py
View file

@ -281,6 +281,9 @@ class TestFlush:
0,
"medium",
"anthropic/claude-opus-5",
0,
0.0,
0.0,
)
def test_a_connect_error_retries_the_same_statement(self):
@ -307,7 +310,20 @@ class TestFlush:
class TestEnqueueSeam:
@pytest.mark.asyncio
@pytest.mark.parametrize("classifier_cost", [0.005, 0.0, None])
async def test_update_database_seam_enqueues_only_auto_routed_success(self, classifier_cost: float | None):
@pytest.mark.parametrize("estimate, covered, saved", [
({"version": 1, "status": "estimated"}, 1, -0.003),
({"version": 1, "status": "estimated"}, 1, 0.0),
({"version": 2, "status": "estimated"}, 1, 0.0),
({"version": 3, "status": "estimated"}, 1, -0.003),
({"version": 1, "status": "unknown"}, 0, 0.0),
({"version": 0, "status": "estimated"}, 0, 0.0),
({"version": 4, "status": "estimated"}, 0, 0.0),
({"version": True, "status": "estimated"}, 0, 0.0),
(None, 0, -0.003),
])
async def test_update_database_seam_enqueues_only_auto_routed_success(
self, classifier_cost: float | None, estimate: dict[str, object] | None, covered: int, saved: float,
) -> None:
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
writer: Final = DBSpendUpdateWriter()
@ -315,7 +331,8 @@ class TestEnqueueSeam:
_autorouter_turn_transactions_lock=asyncio.Lock(), autorouter_turn_transactions=[]
)
metadata: Final = _metadata(
routing_decision={**ROUTING_DECISION, "classifier_cost": classifier_cost}, autorouter_savings=-0.003
routing_decision={**ROUTING_DECISION, "classifier_cost": classifier_cost},
autorouter_savings=saved if covered else -0.003, autorouter_savings_estimate=estimate,
)
for payload in (
_payload(metadata=json.dumps(metadata)),
@ -330,7 +347,10 @@ class TestEnqueueSeam:
assert transaction.router_name == "live-auto"
assert transaction.spend == pytest.approx(0.01 + (classifier_cost or 0.0))
assert transaction.classifier_cost == (classifier_cost or 0.0)
assert transaction.saved_spend == -0.003
assert transaction.saved_spend == saved
assert transaction.savings_estimated_turns == covered
assert transaction.savings_estimated_actual_spend == pytest.approx(transaction.spend if covered else 0.0)
assert transaction.savings_estimated_saved_spend == (saved if covered else 0.0)
def test_every_drain_trigger_reads_the_one_queue_census_owner():

22
tests/test_litellm/proxy/db/test_db_spend_update_writer.py
View file

@ -2492,7 +2492,18 @@ async def test_daily_transaction_carries_compression_saved_tokens():
@pytest.mark.asyncio
async def test_daily_transaction_compression_saved_tokens_zero_when_absent():
@pytest.mark.parametrize("estimate, recorded_savings, expected", [
pytest.param(None, None, -0.005, id="plain-classifier-cost"),
pytest.param({"version": 1, "status": "unknown"}, None, 0.0, id="unknown"),
pytest.param({"version": 2, "status": "unknown"}, None, 0.0, id="unknown-v2"),
pytest.param({"version": 1, "status": "unknown"}, -0.003, 0.0, id="unknown-stale-value"),
pytest.param({"version": 0, "status": "estimated"}, -0.003, 0.0, id="unsupported-version"),
pytest.param({"version": 1, "status": "estimated"}, -0.003, -0.003, id="estimated"),
pytest.param(None, -0.003, -0.003, id="legacy"),
])
async def test_daily_transaction_compression_saved_tokens_zero_when_absent(
estimate: dict[str, object] | None, recorded_savings: float | None, expected: float,
) -> None:
"""Requests without any compression metadata produce a zero count."""
writer = DBSpendUpdateWriter()
mock_prisma = MagicMock()
@ -2510,7 +2521,12 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent():
"prompt_tokens": 100,
"completion_tokens": 10,
"spend": 0.01,
"metadata": json.dumps({"usage_object": {}}),
"metadata": json.dumps({
"usage_object": {"prompt_tokens": 100, "completion_tokens": 10},
"routing_decision": {"savings_baseline_model": "anthropic/claude-sonnet-5", "classifier_cost": 0.005},
"autorouter_savings": recorded_savings,
"autorouter_savings_estimate": estimate,
}),
}
transaction = await writer._common_add_spend_log_transaction_to_daily_transaction(
@ -2523,6 +2539,8 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent():
assert transaction["compression_saved_tokens"] == 0
assert transaction["compression_savings_spend"] == 0
assert transaction["prompt_caching_savings_spend"] == 0
assert transaction["spend"] == 0.01
assert transaction["autorouter_savings_spend"] == expected
# ---------------------------------------------------------------------------

326
tests/test_litellm/proxy/hooks/test_autorouter_baseline_cache.py Normal file
View file

@ -0,0 +1,326 @@
import asyncio
import json
from collections.abc import AsyncIterator, Callable, Generator, Mapping
from contextlib import contextmanager
from datetime import datetime
from types import MappingProxyType
from typing import Final, cast
from uuid import uuid4
import httpx
import pytest
import respx
from pydantic import JsonValue, TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.anthropic.prompt_cache_prediction import NativePredictionTarget, TokenCounter
from litellm.proxy.hooks.autorouter_baseline_cache import AutoRouterBaselineCache, CapturedBaselineObservation
from litellm.router import Router
from litellm.types.router import RetryPolicy
from litellm.types.utils import CallTypes, StandardLoggingRoutingDecision
pytestmark: Final = pytest.mark.asyncio
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_OBJECTS: Final = TypeAdapter(dict[str, object])
_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]])
_MESSAGES_JSON: Final = """[{"role":"user","content":[
{"type":"text","text":"stable","cache_control":{"type":"ephemeral","ttl":"1h"}},
{"type":"text","text":"question"}]}]"""
_MODELS: Final = _MESSAGES.validate_json("""[
{"model_name":"test-router","litellm_params":{"model":"auto_router/complexity_router",
"complexity_router_config":{"tiers":{"SIMPLE":"sonnet","MEDIUM":"sonnet","COMPLEX":"sonnet",
"REASONING":"opus"},"session_affinity":false,
"keyword_tier_rules":[{"keywords":["USE_OPUS"],"tier":"REASONING"}]}}},
{"model_name":"sonnet","litellm_params":{"model":"anthropic/claude-sonnet-5","api_key":"test-selected"},
"model_info":{"id":"selected"}},
{"model_name":"opus","litellm_params":{"model":"anthropic/claude-opus-5","api_key":"test-selected"},
"model_info":{"id":"baseline"}}]""")
def _message(completed: bool, model: str) -> Mapping[str, JsonValue]:
return _JSON_OBJECT.validate_json(f"""{{
"id":"msg_baseline_test","type":"message","role":"assistant","model":{json.dumps(model)},
"content":{'[{"type":"text","text":"OK"}]' if completed else "[]"},
"stop_reason":{'"end_turn"' if completed else "null"},"stop_sequence":null,
"usage":{{"input_tokens":1000,"output_tokens":{10 if completed else 0},
"cache_creation_input_tokens":5000,"cache_read_input_tokens":0,
"cache_creation":{{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":5000}}}}}}""")
_EVENTS: Final = _MESSAGES.validate_json("""[
{"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}},
{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"OK"}},
{"type":"content_block_stop","index":0},
{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":10}},
{"type":"message_stop"}
]""")
async def _count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int:
assert model == "claude-opus-5"
return 6000 if "question" in json.dumps(_JSON_OBJECT.validate_python(body)) else 5000
class _CallContext(TypedDict):
litellm_logging_obj: NotRequired[ReadOnly[Logging]]
litellm_call_id: ReadOnly[str]
litellm_metadata: ReadOnly[Mapping[str, object]]
litellm_session_id: ReadOnly[str]
def _kwargs(logging_obj: Logging, trusted: bool = True, *, explicit_logging: bool = True) -> _CallContext:
context: Final = _OBJECTS.validate_json('{"litellm_metadata":{"user_api_key_hash":"test-caller-hash"}}')
Router._record_routing_decision( # pyright: ignore[reportUnknownMemberType, reportPrivateUsage] # production trusted stamp owner
context,
StandardLoggingRoutingDecision(
router_model_name="test-router",
router_type="complexity",
routed_model="sonnet",
cause="heuristic_scorer",
conversation_continuing=True,
savings_baseline_model="anthropic/claude-opus-5",
savings_baseline_deployment_id="baseline",
),
)
metadata: Final = _OBJECTS.validate_python(context["litellm_metadata"])
if not trusted:
metadata["_autorouter_baseline_route"] = _JSON_OBJECT.validate_json(
'{"router_name":"test-router","baseline_model":"anthropic/claude-opus-5","baseline_deployment_id":"baseline"}'
)
envelope: Final[_CallContext] = {
"litellm_call_id": logging_obj.litellm_call_id,
"litellm_session_id": "baseline-session",
"litellm_metadata": metadata,
}
supplied: Final[_CallContext] = {**envelope, "litellm_logging_obj": logging_obj}
return supplied if explicit_logging else envelope
def _stream(logging_obj: Logging) -> bool:
return logging_obj.stream is True # pyright: ignore[reportUnknownMemberType] # normalize the legacy Logging flag
def _sse(completed: bool = True, model: str = "claude-sonnet-5") -> tuple[bytes, ...]:
events: Final = (
{ # mutable-ok: json.dumps needs a concrete event dictionary
"type": "message_start",
"message": _message(False, model),
},
*_EVENTS,
)
return tuple(
f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode()
for event in (events if completed else events[:-1])
)
def _upstream(request: httpx.Request) -> httpx.Response:
body: Final = _JSON_OBJECT.validate_json(request.content)
model: Final = body.get("model")
assert isinstance(model, str)
stream: Final = body.get("stream") is True
content: Final = b"".join(_sse(model=model)) if stream else json.dumps(_message(True, model)).encode()
return httpx.Response(200, content=content, request=request,
headers=MappingProxyType({"content-type": "text/event-stream" if stream else "application/json"}),
)
def _error(request: httpx.Request, code: int, message: str) -> httpx.Response:
return httpx.Response(
code,
text='{"type":"error","error":{"type":"rate_limit_error","message":' + json.dumps(message) + "}}",
headers=MappingProxyType({"retry-after": "0"}),
request=request,
)
@contextmanager
def _transport(upstream: Callable[[httpx.Request], httpx.Response]) -> Generator[respx.Route]:
with respx.mock() as transport:
yield transport.post("https://api.anthropic.com/v1/messages").mock(side_effect=upstream)
class _NativeOptions(TypedDict):
api_key: NotRequired[ReadOnly[str]]
num_retries: NotRequired[ReadOnly[int]]
async def _call(
target: Router | None,
logging_obj: Logging,
*,
trusted: bool = True,
messages: str = _MESSAGES_JSON,
explicit_logging: bool = True,
) -> None:
invoke: Final = target.anthropic_messages if target else litellm.anthropic_messages # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # legacy native call signatures
options: Final = _NativeOptions() if target else _NativeOptions(api_key="test-selected", num_retries=0)
response: Final[object] = await invoke( # pyright: ignore[reportUnknownVariableType] # native Router returns an opaque SDK result
model="test-router" if target else "anthropic/claude-sonnet-5",
max_tokens=16,
stream=_stream(logging_obj),
messages=_MESSAGES.validate_json(messages),
**options,
**_kwargs(logging_obj, trusted, explicit_logging=explicit_logging),
)
assert response is not None
if _stream(logging_obj):
assert isinstance(response, AsyncIterator)
stream: Final = cast(AsyncIterator[object], response) # cast-ok: iterator checked; all items satisfy object
assert tuple([chunk async for chunk in stream])
class _Capture(CustomLogger):
def __init__(self, call_id: str) -> None:
self.call_id: Final = call_id
self.payloads: Final[asyncio.Queue[Mapping[str, object]]] = asyncio.Queue()
async def async_log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
) -> None:
payload: Final = _OBJECTS.validate_python(kwargs.get("standard_logging_object"))
if payload.get("litellm_call_id") == self.call_id:
self.payloads.put_nowait(payload)
async def payload(self) -> Mapping[str, object]:
return await asyncio.wait_for(self.payloads.get(), timeout=20)
class _Rig:
def __init__(self, monkeypatch: pytest.MonkeyPatch, *, retries: int = 0, count: TokenCounter = _count) -> None:
self.router: Final = Router(model_list=_MODELS, num_retries=retries,
retry_policy=RetryPolicy(RateLimitErrorRetries=retries), disable_cooldowns=True)
def router() -> Router:
return self.router
self.hook: Final = AutoRouterBaselineCache(None, router=router, token_counter=count)
self.call_id: Final = uuid4().hex
self.capture: Final = _Capture(self.call_id)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
for name in ("ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"):
monkeypatch.delenv(name, raising=False)
monkeypatch.setattr(litellm, "callbacks", [self.hook])
for name in ("success_callback", "failure_callback", "_async_failure_callback"):
monkeypatch.setattr(litellm, name, [])
monkeypatch.setattr(litellm, "_async_success_callback", [self.capture])
def logging(self, stream: bool = False) -> Logging:
return Logging(model="anthropic/claude-sonnet-5", messages=_MESSAGES.validate_json(_MESSAGES_JSON),
stream=stream, call_type=CallTypes.anthropic_messages.value, start_time=datetime.now(),
litellm_call_id=self.call_id, function_id=self.call_id, kwargs={"litellm_session_id":"baseline-session"})
def _observation(payload: Mapping[str, object]) -> CapturedBaselineObservation:
encoded: Final = payload["autorouter_baseline_observation"]
assert isinstance(encoded, str)
assert "test-selected" not in encoded and "stable" not in encoded and "x-api-key" not in encoded
return CapturedBaselineObservation.model_validate_json(encoded)
@pytest.mark.parametrize("stream,baseline", ((False, False), (True, False), (False, True), (True, True)))
async def test_native_logging_captures_usage_without_publishing_hypothetical_savings(
monkeypatch: pytest.MonkeyPatch, stream: bool, baseline: bool,
) -> None:
rig: Final = _Rig(monkeypatch)
messages: Final = _MESSAGES_JSON.replace("question", "question USE_OPUS") if baseline else _MESSAGES_JSON
with _transport(_upstream):
await _call(rig.router, rig.logging(stream), messages=messages)
payload: Final = await rig.capture.payload()
captured: Final = _observation(payload)
assert payload["autorouter_savings"] is None
assert _OBJECTS.validate_python(payload["autorouter_savings_estimate"])["reason"] == "pending_projection"
assert captured.observation.outcome == "complete"
assert captured.observation.baseline_equivalent == baseline
assert captured.observation.usage is not None and captured.observation.usage.completion_tokens == 10
assert captured.observation.plan is not None and captured.observation.plan.total_tokens == 6000
async def test_count_failure_preserves_initial_observed_equivalence(monkeypatch: pytest.MonkeyPatch) -> None:
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int | None:
return None
rig: Final = _Rig(monkeypatch, count=count)
with _transport(_upstream):
await _call(rig.router, rig.logging(), messages=_MESSAGES_JSON.replace("question", "question USE_OPUS"))
captured: Final = _observation(await rig.capture.payload())
assert captured.observation.baseline_equivalent and captured.observation.usage is not None
assert captured.observation.plan is None and captured.observation.reason == "token_count_unavailable"
async def test_native_retry_is_uncertain_even_when_final_response_succeeds(monkeypatch: pytest.MonkeyPatch) -> None:
rig: Final = _Rig(monkeypatch, retries=1)
def upstream(request: httpx.Request) -> httpx.Response:
return _upstream(request) if route.call_count else _error(request, 429, "retry")
with _transport(upstream) as route:
await _call(rig.router, rig.logging())
captured: Final = _observation(await rig.capture.payload())
assert route.call_count == 2
assert captured.observation.outcome == "uncertain"
assert captured.observation.reason == "retried_request"
async def test_caller_cannot_forge_an_observation_scope(monkeypatch: pytest.MonkeyPatch) -> None:
rig: Final = _Rig(monkeypatch)
with _transport(_upstream):
await _call(None, rig.logging(), trusted=False)
payload: Final = await rig.capture.payload()
assert payload["autorouter_baseline_observation"] is None
assert payload["autorouter_savings"] is None
@pytest.mark.parametrize("model,key,endpoint", (
("claude-sonnet-5", "test-first", None),
("claude-opus-5", "test-second", None),
("claude-opus-5", "test-first", "https://example.test"),
))
async def test_count_memo_is_scoped_to_provider_recipient(model: str, key: str, endpoint: str | None) -> None:
counts: Final = iter((5000, 6000))
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int:
return next(counts)
collector: Final = AutoRouterBaselineCache(None, token_counter=count)
original: Final = NativePredictionTarget("claude-opus-5", "test-first")
other: Final = NativePredictionTarget(model, key, endpoint)
assert await collector._count(original, {}) == 5000 # pyright: ignore[reportPrivateUsage]
assert await collector._count(other, {}) == 6000 # pyright: ignore[reportPrivateUsage]
assert await collector._count(original, {}) == 5000 # pyright: ignore[reportPrivateUsage]
@pytest.mark.parametrize("stream", (False, True))
async def test_provider_counting_does_not_hold_the_inference_response(
monkeypatch: pytest.MonkeyPatch, stream: bool,
) -> None:
counting: Final = asyncio.Event()
release: Final = asyncio.Event()
async def count(model: str, api_key: str, body: Mapping[str, JsonValue]) -> int:
counting.set()
await release.wait()
return await _count(model, api_key, body)
rig: Final = _Rig(monkeypatch, count=count)
try:
with _transport(_upstream):
await asyncio.wait_for(_call(rig.router, rig.logging(stream)), timeout=2)
await asyncio.wait_for(counting.wait(), timeout=2)
assert rig.capture.payloads.empty()
release.set()
assert _observation(await rig.capture.payload()).observation.plan is not None
finally:
release.set()

76
tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py
View file

@ -546,6 +546,9 @@ class TestAutoRouterBenchmarks:
total_tokens=4000,
spend=10.0,
saved_spend=30.0,
savings_estimated_turns=40,
savings_estimated_actual_spend=10.0,
savings_estimated_saved_spend=30.0,
classifier_cost=0.4,
classifier_cost_recorded_turns=40,
session_seconds=400.0,
@ -582,12 +585,29 @@ class TestAutoRouterBenchmarks:
def test_a_losing_router_reports_negative_savings(self):
from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals
losing = self.ROW.model_copy(update={"saved_spend": -5.0})
losing = self.ROW.model_copy(update={"saved_spend": -5.0, "savings_estimated_saved_spend": -5.0})
totals = _benchmark_totals(losing)
assert totals.baseline_spend == 5.0
assert totals.saved_pct == -100.0
assert totals.classifier_cost == 0.4
@pytest.mark.parametrize("estimated_turns", [0, 4])
def test_savings_compare_only_the_current_estimated_cohort(self, estimated_turns: int) -> None:
from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals
row: Final = self.ROW.model_copy(update={
"savings_estimated_turns": estimated_turns,
"savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0,
"savings_estimated_saved_spend": -0.5 if estimated_turns else 0.0,
})
totals: Final = _benchmark_totals(row)
assert totals.spend == 10.0
assert totals.savings_estimated_turns == estimated_turns
assert totals.saved_spend == (-0.5 if estimated_turns else None)
assert totals.baseline_spend == (1.5 if estimated_turns else None)
assert totals.saved_pct == (pytest.approx(-33.3) if estimated_turns else None)
assert totals.saved_per_session is None
def test_an_empty_window_folds_to_zeros(self):
from litellm.proxy.management_endpoints.auto_router_endpoints import (
_benchmark_totals,
@ -607,7 +627,10 @@ class TestAutoRouterBenchmarks:
_summed_agg_row,
)
other = self.ROW.model_copy(update={"router_name": "auto-2", "sessions": 1, "turns": 10, "spend": 0.0})
other = self.ROW.model_copy(update={
"router_name": "auto-2", "sessions": 1, "turns": 10, "spend": 0.0,
"savings_estimated_turns": 10, "savings_estimated_actual_spend": 0.0,
})
summed = _summed_agg_row([self.ROW, other])
totals = _benchmark_totals(summed)
assert summed.sessions == 5
@ -696,6 +719,9 @@ class TestAutoRouterBenchmarks:
"turns": 10,
"spend": 2.0,
"saved_spend": -0.5,
"savings_estimated_turns": 10,
"savings_estimated_actual_spend": 2.0,
"savings_estimated_saved_spend": -0.5,
"classifier_cost": recorded_turns * 0.02,
"classifier_cost_recorded_turns": recorded_turns,
}
@ -876,6 +902,10 @@ class TestAutoRouterSession:
"last_model": "anthropic/claude-sonnet-5",
"spend": 0.14,
"saved_spend": 0.24,
"savings_estimated_turns": 3,
"savings_estimated_actual_spend": 0.14,
"savings_estimated_saved_spend": 0.24,
"savings_estimated_baseline_models": {"anthropic/claude-opus-5": 3},
"classifier_cost": 0.0,
"tier_turns": {"simple": 1, "complex": 2},
"baseline_models": {"anthropic/claude-opus-5": 3},
@ -899,25 +929,33 @@ class TestAutoRouterSession:
return lookups
@pytest.mark.asyncio
@pytest.mark.parametrize("turns, estimated", [(3, True), (10, True), (10, False)], ids=["full", "partial", "legacy"])
async def test_a_key_reads_its_own_session_with_the_baseline_its_turns_were_priced_against(
self, monkeypatch: pytest.MonkeyPatch
):
self, monkeypatch: pytest.MonkeyPatch, turns: int, estimated: bool,
) -> None:
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session
caller = UserAPIKeyAuth(api_key="sk-caller")
self._rig(monkeypatch, [{**self.ROW, "api_key": caller.api_key, "session_id": "sess-1"}])
row: Final = {key: value for key, value in self.ROW.items() if estimated or not key.startswith("savings_estimated_")}
spend: Final = 0.14 if turns == 3 else 10.0
if estimated and turns != 3:
row["savings_estimated_saved_spend"] = -0.04
self._rig(monkeypatch, [{**row, "api_key": caller.api_key, "session_id": "sess-1", "turns": turns, "spend": spend}])
response = await get_auto_router_session(user_api_key_dict=caller, session_id="sess-1")
assert response.model_dump() == {
"session_id": "sess-1",
"router_name": "claude-auto",
"router_type": "complexity",
"turns": 3,
"turns": turns,
"last_model": "anthropic/claude-sonnet-5",
"spend": 0.14,
"saved_spend": 0.24,
"baseline_spend": pytest.approx(0.38),
"baseline_model": "anthropic/claude-opus-5",
"baseline_models": {"anthropic/claude-opus-5": 3},
"spend": spend,
"saved_spend": (0.24 if turns == 3 else -0.04) if estimated else None,
"savings_estimated_turns": 3 if estimated else 0,
"savings_estimated_actual_spend": 0.14 if estimated else 0.0,
"baseline_spend": pytest.approx(0.38) if turns == 3 else None,
"savings_estimated_baseline_spend": pytest.approx(0.38 if turns == 3 else 0.1) if estimated else None,
"baseline_model": "anthropic/claude-opus-5" if estimated else None,
"baseline_models": {"anthropic/claude-opus-5": 3} if estimated else {},
}
@pytest.mark.asyncio
@ -959,22 +997,14 @@ class TestAutoRouterSession:
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session
priced = {"anthropic/claude-opus-5": 2, "anthropic/claude-sonnet-5": 1}
self._rig(monkeypatch, [{**self.ROW, "api_key": ADMIN.api_key, "session_id": "s", "baseline_models": priced}])
self._rig(monkeypatch, [{
**self.ROW, "api_key": ADMIN.api_key, "session_id": "s",
"baseline_models": {"old-baseline": 100}, "savings_estimated_baseline_models": priced,
}])
response = await get_auto_router_session(user_api_key_dict=ADMIN, session_id="s")
assert response.baseline_model == "anthropic/claude-opus-5"
assert response.baseline_models == priced
@pytest.mark.asyncio
async def test_a_session_whose_turns_recorded_no_baseline_reports_the_money_without_a_name(
self, monkeypatch: pytest.MonkeyPatch
):
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session
self._rig(monkeypatch, [{**self.ROW, "api_key": ADMIN.api_key, "session_id": "s", "baseline_models": {}}])
response = await get_auto_router_session(user_api_key_dict=ADMIN, session_id="s")
assert response.baseline_model is None
assert response.baseline_spend == pytest.approx(0.38)
@pytest.mark.asyncio
async def test_an_oversized_client_session_id_is_bounded_like_the_writer_bounded_it(
self, monkeypatch: pytest.MonkeyPatch

199
tests/test_litellm/proxy/spend_tracking/test_baseline_accounting.py Normal file
View file

@ -0,0 +1,199 @@
from dataclasses import replace
from itertools import groupby
from typing import Final
import pytest
import litellm
from litellm.llms.anthropic.cost_calculation import cost_per_token
from litellm.llms.anthropic.prompt_cache_prediction import CountedBreakpoint, CountedPromptCachePlan
from litellm.proxy.spend_tracking.baseline_accounting import (
BaselineEstimate,
BaselineHistory,
BaselineObservation,
CacheEntry,
advance_baseline_history,
)
from litellm.types.utils import CacheCreationTokenDetails, PromptTokensDetailsWrapper, Usage
def _usage() -> Usage:
return Usage(
prompt_tokens=6200,
completion_tokens=30,
total_tokens=6230,
cache_read_input_tokens=0,
cache_creation_input_tokens=6000,
speed="fast",
inference_geo="us",
completion_tokens_details={"reasoning_tokens": 20},
server_tool_use={"web_search_requests": 1},
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=200,
cached_tokens=0,
cache_creation_tokens=6000,
cache_write_tokens=6000,
cache_creation_token_details=CacheCreationTokenDetails(
ephemeral_5m_input_tokens=0, ephemeral_1h_input_tokens=6000
),
),
)
def _marker(
name: str = "prefix", ttl: int = 3600, tokens: int = 6000, previous: tuple[str, ...] = ()
) -> CountedBreakpoint:
return CountedBreakpoint(
fingerprint=f"{name}:{ttl}",
ttl_seconds=ttl,
prefix_tokens=tokens,
lookback_fingerprints=(*(f"{item}:{ttl}" for item in previous), f"{name}:{ttl}"),
content_fingerprint=name,
lookback_content_fingerprints=(*previous, name),
)
def _observation(request_id: str, started: float = 10000.0, **overrides: object) -> BaselineObservation:
return BaselineObservation.model_validate(
{
"request_id": request_id,
"started_at": started,
"available_at": started + 0.1,
"outcome": "complete",
"baseline_equivalent": False,
"usage": _usage(),
"plan": CountedPromptCachePlan(6200, (_marker(),)),
"minimum_cache_tokens": 4096,
**overrides,
}
)
def _replay(*observations: BaselineObservation) -> tuple[BaselineEstimate, ...]:
history = BaselineHistory()
results: list[BaselineEstimate] = []
for _, group in groupby(sorted(observations, key=lambda item: item.started_at), key=lambda item: item.started_at):
history, estimates = advance_baseline_history(history, tuple(group))
results.extend(estimates)
return tuple(results)
def test_initial_identical_path_preserves_full_usage_without_counting_or_exclusive_owner() -> None:
initial: Final = _observation("main", baseline_equivalent=True, plan=None, reason="unsupported_request_headers")
background: Final = initial.model_copy(update={"request_id": "background"})
later: Final = initial.model_copy(update={"request_id": "later", "started_at": 10001.0, "available_at": 10002.0})
estimates: Final = _replay(initial, background, later)
assert all(item.provenance == "observed_identical" and item.usage == initial.usage for item in estimates)
assert all(item.usage is not initial.usage for item in estimates)
assert all(item.usage.prompt_tokens == 6200 for item in estimates if item.usage is not None)
def test_late_divergent_observation_replays_in_event_order_and_removes_initial_zero() -> None:
same: Final = _observation("same", 10001.0, baseline_equivalent=True)
early: Final = _observation("early")
assert _replay(same)[0].provenance == "observed_identical"
replayed: Final = _replay(same, early)
assert replayed == _replay(early, same)
assert replayed[0].usage is None
assert replayed[1].provenance == "modeled"
assert replayed[1].usage is not None and replayed[1].usage.prompt_tokens_details.cached_tokens == 6000
@pytest.mark.parametrize("ttl", [300, 3600])
def test_prefix_match_expiry_and_usage_pricing_fields(ttl: int) -> None:
plan: Final = CountedPromptCachePlan(6200, (_marker(ttl=ttl),))
first: Final = _observation("first", baseline_equivalent=True, plan=plan)
# Each replay starts from the original observation, so warm does not refresh the expiry case.
warm: Final = _replay(first, _observation("warm", 10000.0 + ttl - 0.01, plan=plan))[-1]
cold: Final = _replay(first, _observation("cold", 10000.0 + ttl, plan=plan))[-1]
assert warm.reason == "cache_prefix_available" and cold.reason == "cache_prefix_expired"
assert warm.usage is not None and cold.usage is not None
assert warm.usage.prompt_tokens_details.cached_tokens == 6000
assert cold.usage.prompt_tokens_details.cached_tokens == 0
assert cold.usage.prompt_tokens_details.cache_creation_tokens == 6000
unaffected: Final = {"prompt_tokens", "total_tokens", "prompt_tokens_details", "cache_read_input_tokens", "cache_creation_input_tokens"}
assert warm.usage.model_dump(exclude=unaffected) == first.usage.model_dump(exclude=unaffected)
assert cold.usage.model_dump(exclude=unaffected) == first.usage.model_dump(exclude=unaffected)
@pytest.mark.parametrize("warm_tail", (False, True))
def test_growth_lookback_and_mixed_ttl_keep_distinct_read_write_buckets(warm_tail: bool) -> None:
first: Final = _observation("first", baseline_equivalent=True)
grown: Final = CountedPromptCachePlan(7100, (_marker("grown", 3600, 6500, ("prefix",)), _marker("tail", 300, 7000)))
# Initial unseen suffixes remain unknown within their potential pre-existing cache horizon.
second: Final = _replay(first, _observation("second", 10001.0, plan=grown))[-1]
assert second.reason == "history_unavailable"
history: Final = BaselineHistory(
first_at=1.0, last_at=10000.0, equivalent=False, uncertain_before=1.0,
entries=(CacheEntry("tail:300", "tail", 7000, 300, 10000.0, 10300.0),) if warm_tail else (),
)
_, estimates = advance_baseline_history(history, (_observation("mixed", 10001.0, plan=grown),))
usage: Final = estimates[0].usage
assert usage is not None
assert usage.prompt_tokens_details.text_tokens == 100
# Anthropic billing locations: B is the highest 1h breakpoint AFTER the highest hit A.
# https://platform.claude.com/docs/en/build-with-claude/prompt-caching#mixing-different-ttls (2026-09-15)
assert usage.prompt_tokens_details.cached_tokens == (7000 if warm_tail else 0)
assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_1h_input_tokens == (0 if warm_tail else 6500)
assert usage.prompt_tokens_details.cache_creation_token_details.ephemeral_5m_input_tokens == (0 if warm_tail else 500)
@pytest.mark.parametrize("change", ["prefix", "ttl", "unavailable", "failed", "response_cache"])
def test_uncertainty_and_replays_do_not_manufacture_hits(change: str) -> None:
first: Final = _observation("first", baseline_equivalent=True)
changes: Final = {
"prefix": {"plan": CountedPromptCachePlan(6200, (_marker("changed"),))},
"ttl": {"plan": CountedPromptCachePlan(6200, (_marker(ttl=300),))},
"unavailable": {"plan": None, "reason": "token_count_unavailable"},
"failed": {"outcome": "uncertain", "reason": "incomplete_response"},
"response_cache": {"outcome": "response_cache"},
}
second: Final = _observation("second", 10001.0, **changes[change])
third: Final = _observation("third", 10002.0)
middle, result = _replay(first, second, third)[1:]
assert middle.usage is None
if change in ("unavailable", "failed", "ttl"):
assert result.usage is None
else:
assert result.usage is not None and result.usage.prompt_tokens_details.cached_tokens == 6000
def test_first_token_availability_and_simultaneous_divergence_are_conservative() -> None:
slow: Final = _observation("slow", available_at=10002.0, baseline_equivalent=True)
overlap: Final = _observation("overlap", 10001.0)
assert _replay(slow, overlap)[-1].usage is None
assert all(item.provenance != "observed_identical" for item in _replay(slow, _observation("tie")))
def test_invalid_usage_and_invalid_count_plan_cannot_seed_cache() -> None:
bad: Final = _observation("bad", baseline_equivalent=True, usage=_usage().model_copy(update={"total_tokens": 1}))
assert all(item.usage is None for item in _replay(bad, _observation("next", 10001.0)))
broken: Final = CountedPromptCachePlan(6200, (replace(_marker(), prefix_tokens=7000),))
assert _replay(_observation("bad", plan=broken))[0].usage is None
def test_overlapping_uncertain_request_cannot_be_warmed_by_a_later_callback() -> None:
uncertain: Final = _observation("incomplete", outcome="uncertain", available_at=10010.0)
overlap: Final = _observation("overlap", 10001.0)
during: Final = _observation("during", 10002.0)
after: Final = _observation("after", 10011.0)
warmed: Final = _observation("warmed", 10012.0)
estimates: Final = _replay(uncertain, overlap, during, after, warmed)
assert estimates[1].reason == estimates[2].reason == "concurrent_uncertainty"
assert estimates[3].usage is None
assert estimates[4].usage is not None and estimates[4].usage.prompt_tokens_details.cached_tokens == 6000
def test_modeled_read_cannot_recharge_the_original_private_write_count() -> None:
warm: Final = _replay(_observation("initial", baseline_equivalent=True), _observation("warm", 10001.0))[-1]
assert warm.usage is not None
prices: Final = {
**litellm.get_model_info("claude-opus-5", custom_llm_provider="anthropic"),
"input_cost_per_token": 1e-6,
"output_cost_per_token": 2e-6,
"cache_read_input_token_cost": 1e-7,
"cache_creation_input_token_cost": 1.25e-6,
"provider_specific_entry": {"fast": 2.0, "us": 1.1},
}
input_cost, output_cost = cost_per_token("claude-opus-5", warm.usage, model_info=prices)
assert input_cost + output_cost == pytest.approx((200 * 1e-6 + 6000 * 1e-7 + 30 * 2e-6) * 2.0 * 1.1)

415
tests/test_litellm/proxy/spend_tracking/test_savings.py
View file

@ -1,4 +1,4 @@
from typing import Final
from typing import Final, Literal
import pytest
@ -23,13 +23,13 @@ pytestmark = pytest.mark.usefixtures("local_model_cost_map")
def test_baseline_preserves_anthropic_pricing_fields(modifier: dict[str, str], continuing: bool) -> None:
usage: Final = _usage(1000, 0, 1000, 100).model_copy(update=modifier)
expected: Final = (_usage(1000, 1000, 0, 100) if continuing else usage).model_copy(update=modifier)
normalized: Final = _baseline_usage(usage, continuing)
normalized: Final = _baseline_usage(expected)
cache_fields: Final = {"prompt_tokens_details", "cache_read_input_tokens", "cache_creation_input_tokens"}
assert normalized.model_dump(exclude=cache_fields) == usage.model_dump(exclude=cache_fields)
assert usage.prompt_tokens_details.cached_tokens == 0
selected_cost: Final = 0.013
assert compute_autorouter_savings(
"claude-opus-5", "claude-sonnet-5", "anthropic", usage, conversation_continuing=continuing,
"claude-opus-5", "claude-sonnet-5", "anthropic", usage, baseline_usage=expected,
cost_breakdown={"input_cost": 0.01, "output_cost": 0.003},
) == pytest.approx(sum(anthropic_cost_per_token("claude-opus-5", expected)) - selected_cost)
@ -41,7 +41,7 @@ def test_anthropic_baseline_keeps_negotiated_prices_with_provider_multiplier() -
}
usage: Final = _usage(1000, 1000, 0, 100).model_copy(update={"speed": "fast"})
assert compute_autorouter_savings(
"claude-opus-5", "claude-sonnet-5", "anthropic", usage, baseline_info=info,
"claude-opus-5", "claude-sonnet-5", "anthropic", usage, baseline_info=info, baseline_usage=usage,
cost_breakdown={"input_cost": 0.01, "output_cost": 0.003},
) == pytest.approx(0.0015 * 2 - 0.013)
@ -428,146 +428,109 @@ def test_negative_token_counts_clamp_to_zero():
assert result.prompt_caching == 0.0
def _usage(fresh: int, cached: int, written: int, out: int) -> Usage:
def _usage(fresh: int, cached: int, written: int, out: int, *, hour: bool = False, image: int = 0) -> Usage:
"""Usage as the spend log records it; `prompt_tokens` is the inclusive total."""
return Usage(
prompt_tokens=fresh + cached + written,
completion_tokens=out,
total_tokens=fresh + cached + written + out,
prompt_tokens_details={"cached_tokens": cached, "cache_creation_tokens": written, "text_tokens": fresh},
prompt_tokens_details={
"cached_tokens": cached, "cache_creation_tokens": written, "text_tokens": fresh - image, "image_tokens": image,
"cache_creation_token_details": {"ephemeral_1h_input_tokens": written} if hour else None,
},
cache_read_input_tokens=cached,
cache_creation_input_tokens=written,
)
def _savings(baseline: str, selected: str, usage: Usage, continuing: bool = True) -> float:
"""Savings for a request, defaulting to a conversation already underway.
`continuing=True` is the mid-conversation case, where the baseline had the prompt
cached and this request's write is what the switch cost. `continuing=False` is a
conversation's first turn, where nothing was cached for any model.
"""
def _savings(baseline: str, selected: str, usage: Usage, baseline_usage: Usage | None = None) -> float | None:
return compute_autorouter_savings(
baseline_model=baseline,
selected_model=selected,
selected_provider="anthropic",
usage=usage,
conversation_continuing=continuing,
baseline_usage=baseline_usage,
)
def test_switching_models_mid_conversation_charges_the_cold_cache_write():
"""Staying on one model writes the cache once and reads it thereafter. Switching
leaves the new model cold, so it pays to write the whole prompt again; when that
charge outweighs the cheaper rates the route lost money and must report a loss.
Pricing the baseline as if it too re-wrote the cache credits a charge it never
paid, which is how a losing switch used to read as the largest saving on the page.
"""
usage = _usage(fresh=3, cached=500, written=12304, out=500)
result = _savings("claude-sonnet-5", "claude-haiku-4-5", usage)
sonnet = litellm.get_model_info("claude-sonnet-5", "anthropic")
haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic")
warm_baseline = (
3 * sonnet["input_cost_per_token"]
+ 12804 * sonnet["cache_read_input_token_cost"]
+ 500 * sonnet["output_cost_per_token"]
@pytest.mark.parametrize("baseline, selected, actual, modeled, loses_money", [
pytest.param("claude-sonnet-5", "claude-haiku-4-5", _usage(3, 500, 12304, 500),
_usage(3, 12804, 0, 500), True, id="warm-baseline-cold-route"),
pytest.param("claude-opus-5", "claude-opus-5", _usage(0, 0, 20000, 1000),
_usage(0, 20000, 0, 1000), True, id="same-model-cold-route"),
pytest.param("claude-opus-5", "claude-sonnet-5", _usage(0, 19000, 1000, 1000),
_usage(0, 19500, 500, 1000), False, id="partly-cached-growth"),
pytest.param("claude-opus-5", "claude-sonnet-5", _usage(0, 0, 100000, 1000, hour=True),
_usage(0, 0, 100000, 1000, hour=True), False, id="expired-one-hour"),
pytest.param("claude-opus-5", "claude-sonnet-5", _usage(0, 0, 100000, 1000, hour=True),
_usage(0, 100000, 0, 1000), True, id="invented-one-hour-hit"),
pytest.param("claude-opus-5", "claude-sonnet-5", _usage(4000, 0, 16000, 1000, hour=True, image=4000),
_usage(4000, 0, 16000, 1000, hour=True, image=4000), False, id="image-and-one-hour-write"),
])
def test_supplied_baseline_usage_is_priced_independently(
baseline: str, selected: str, actual: Usage, modeled: Usage, loses_money: bool,
) -> None:
result: Final = _savings(baseline, selected, actual, modeled)
expected: Final = sum(generic_cost_per_token(model=baseline, usage=modeled, custom_llm_provider="anthropic")) - sum(
generic_cost_per_token(model=selected, usage=actual, custom_llm_provider="anthropic")
)
actually_paid = (
3 * haiku["input_cost_per_token"]
+ 500 * haiku["cache_read_input_token_cost"]
+ 12304 * haiku["cache_creation_input_token_cost"]
+ 500 * haiku["output_cost_per_token"]
)
assert result == pytest.approx(warm_baseline - actually_paid)
assert result < 0, "a cache-thrashing switch must report a loss, not a saving"
phantom = 12304 * sonnet["cache_creation_input_token_cost"]
assert result != pytest.approx(warm_baseline + phantom - actually_paid)
assert result == pytest.approx(expected)
assert result is not None and (result < 0) is loses_money
assert _baseline_usage(modeled).prompt_tokens_details == modeled.prompt_tokens_details
def test_a_cold_switch_never_beats_turning_caching_off():
"""Switching to a cold model makes it write the whole prompt again. That write is a
real cost of switching, so the same traffic must look worse than if caching were off
entirely.
The baseline is priced as a warm cache even though this request read nothing: a
switch reads nothing precisely because the new model's cache is empty, and staying
on one model would have had the prompt cached already. Gating the warm baseline on
a read charged the baseline a write it would never repeat, which made a cold switch
report a larger saving than no caching at all.
"""
cold_switch = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 0, 20_000, 1_000))
caching_off = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(20_000, 0, 0, 1_000))
assert cold_switch < caching_off
opus = litellm.get_model_info("claude-opus-5", "anthropic")
haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic")
warm_baseline = 20_000 * opus["cache_read_input_token_cost"] + 1_000 * opus["output_cost_per_token"]
actually_paid = 20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"]
assert cold_switch == pytest.approx(warm_baseline - actually_paid)
@pytest.mark.parametrize("modifier, multiplier", [({}, 1.0), ({"inference_geo": "us"}, 1.1), ({"speed": "fast"}, 2.0)])
@pytest.mark.parametrize("negotiated", [False, True])
@pytest.mark.parametrize("provenance", [None, "modeled", "observed_initial"])
def test_observed_initial_uses_provider_billing_and_effective_rates(
modifier: dict[str, str], multiplier: float, negotiated: bool,
provenance: Literal["modeled", "observed_initial"] | None,
) -> None:
usage: Final = _usage(1000, 2000, 3000, 100).model_copy(update=modifier)
info: Final = litellm.get_model_info("claude-opus-5", "anthropic").copy()
if negotiated:
info["input_cost_per_token"] = 1e-6
info["output_cost_per_token"] = 2e-6
info["cache_read_input_token_cost"] = 3e-7
info["cache_creation_input_token_cost"] = 4e-6
billed: Final = anthropic_cost_per_token("claude-opus-5", usage, model_info=info)
if negotiated:
assert sum(billed) == pytest.approx(0.0138 * multiplier)
assert compute_autorouter_savings(
"anthropic/claude-opus-5", "claude-opus-5", "anthropic", usage,
selected_info=info, baseline_info=info, baseline_usage=usage,
baseline_deployment_id="same", selected_deployment_id="same",
cost_breakdown={"input_cost": billed[0], "output_cost": billed[1]},
baseline_provenance=provenance,
) == 0.0
def test_moving_one_token_between_cache_buckets_does_not_move_the_answer():
"""A continuing conversation writes a few new tokens and reads the rest. Treating the
presence of a write as the signal for a switch made that ordinary increment flip the
result, so a request reading 19,999 and writing 1 landed somewhere entirely different
from one reading 20,000 and writing none.
"""
reads_nothing = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 0, 20_000, 1_000))
reads_one = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", _usage(0, 1, 19_999, 1_000))
assert reads_one == pytest.approx(reads_nothing, abs=1e-4)
def test_multimodal_prompts_are_priced_on_the_baseline_too():
"""The baseline is this same request met by a warm cache, so every field it was
priced on has to survive. Rebuilding the details from the cache buckets alone
dropped the image and audio counts, which priced the baseline as a text-only
request that never ran and shrank the reported saving on multimodal traffic.
"""
details = {"cached_tokens": 0, "cache_creation_tokens": 16_000, "text_tokens": 0, "image_tokens": 4_000}
with_images = Usage(
prompt_tokens=20_000,
completion_tokens=1_000,
total_tokens=21_000,
prompt_tokens_details=details,
)
baseline = _baseline_usage(with_images, conversation_continuing=True)
assert baseline.prompt_tokens_details.image_tokens == 4_000, "image tokens must survive into the baseline"
opus = litellm.get_model_info("claude-opus-5", "anthropic")
priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic")
text_only = 20_000 * opus["cache_read_input_token_cost"]
assert priced > text_only, "dropping the image tokens undercharges the baseline and hides the saving"
def test_the_baseline_is_never_charged_a_cache_write():
"""Carrying the details through must not carry the 5m/1h creation breakdown with
them. `generic_cost_per_token` charges a creation cost whenever that breakdown is
present, even against a zeroed creation count, which would put the phantom write
back on the baseline for every long-cache request.
"""
long_cache = Usage(
prompt_tokens=20_000,
completion_tokens=1_000,
total_tokens=21_000,
prompt_tokens_details={
"cached_tokens": 0,
"cache_creation_tokens": 20_000,
"text_tokens": 0,
"cache_creation_token_details": {"ephemeral_1h_input_tokens": 20_000},
},
)
baseline = _baseline_usage(long_cache, conversation_continuing=True)
opus = litellm.get_model_info("claude-opus-5", "anthropic")
priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic")
assert priced == pytest.approx(20_000 * opus["cache_read_input_token_cost"]), (
"the baseline reads a warm cache; it never pays to create one"
)
@pytest.mark.parametrize("model, deployment, known, delta", [
("claude-sonnet-5", "same", "observed", 0.0),
("claude-opus-5", "other", "observed", 0.0),
("claude-opus-5", "", "observed", 0.0),
("claude-opus-5", "same", "missing", 0.0),
("claude-opus-5", "same", "different", 0.0),
("claude-opus-5", "same", "observed", 0.01),
("claude-opus-5", "same", "prices", 0.0),
("claude-opus-5", "same", "unbilled", 0.0),
])
def test_initial_provenance_cannot_override_mismatched_evidence(
model: str, deployment: str, known: Literal["observed", "missing", "different", "prices", "unbilled"], delta: float,
) -> None:
usage: Final = _usage(1000, 0, 1000, 100)
billed: Final = anthropic_cost_per_token("claude-opus-5", usage)
info: Final = litellm.get_model_info(model, "anthropic").copy()
if known == "prices":
info["cache_read_input_token_cost"] = 0.001 # No reads here: equal charge alone cannot establish equal rates.
assert compute_autorouter_savings(
"claude-opus-5", model, "anthropic", usage,
baseline_usage=(None if known == "missing" else _usage(1000, 1000, 0, 100) if known == "different" else usage),
selected_info=info,
baseline_provenance="observed_initial",
baseline_deployment_id="same", selected_deployment_id=deployment,
cost_breakdown=None if known == "unbilled" else {"input_cost": billed[0] + delta, "output_cost": billed[1]},
) is None
def test_uncached_request_is_the_plain_rate_difference():
@ -588,11 +551,12 @@ def test_escalation_reports_its_real_cost():
def test_autorouter_savings_zero_when_model_unchanged():
assert _savings("claude-opus-5", "claude-opus-5", _usage(3, 500, 12304, 500)) == 0.0
usage: Final = _usage(3, 500, 12304, 500)
assert _savings("claude-opus-5", "claude-opus-5", usage, usage) == 0.0
def test_autorouter_savings_unknown_baseline_fails_open_to_zero():
assert _savings("totally-made-up-model-xyz", "claude-haiku-4-5", _usage(3, 500, 12304, 500)) == 0.0
def test_autorouter_savings_unknown_baseline_remains_unknown():
assert _savings("totally-made-up-model-xyz", "claude-haiku-4-5", _usage(3, 500, 12304, 500)) is None
def test_autorouter_savings_zero_without_baseline():
@ -607,9 +571,7 @@ def test_autorouter_savings_zero_without_baseline():
assert result.autorouter == 0.0
def test_compute_savings_spend_carries_a_losing_switch_through():
"""The signed value must survive into SavingsSpend; clamping it here would put the
dashboard back to only ever showing gains."""
def test_compute_savings_spend_carries_a_recorded_losing_switch_through():
result = compute_savings_spend(
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
@ -617,6 +579,7 @@ def test_compute_savings_spend_carries_a_losing_switch_through():
gateway_injected_cache=True,
routing_decision={"conversation_continuing": True, "savings_baseline_model": "anthropic/claude-sonnet-5"},
usage_object=_cached_usage_object(),
recorded_autorouter_savings=-0.01,
)
assert result.autorouter < 0
@ -651,18 +614,10 @@ def test_malformed_usage_object_does_not_fail_the_spend_write():
assert result.compression > 0
def test_the_same_deployment_spelled_two_ways_is_not_a_switch():
"""The spend log records a normalized model name while the baseline arrives as the
operator wrote it in config. Comparing the raw strings makes a request that never
changed model look like a switch, and prices one deployment against itself."""
# Must be a cached request: the baseline arm is priced against a warm cache and the
# selected arm against what was actually paid, so treating one deployment as two
# charges it a cold-cache write it never took, inventing a loss on a request that
# never changed model. An uncached request prices identically either way and would
# make this assertion vacuous.
usage = _usage(fresh=3, cached=500, written=12304, out=500)
assert _savings("anthropic/claude-opus-5", "claude-opus-5", usage) == 0.0
assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage) == 0.0
def test_equal_modeled_usage_is_zero_under_equivalent_model_names() -> None:
usage: Final = _usage(3, 500, 12304, 500)
assert _savings("anthropic/claude-opus-5", "claude-opus-5", usage, usage) == 0.0
assert _savings("claude-opus-5", "anthropic/claude-opus-5", usage, usage) == 0.0
def test_baseline_is_priced_under_its_own_provider():
@ -686,99 +641,9 @@ def test_baseline_is_priced_under_its_own_provider():
assert azure > 0 > deepseek
def test_unresolvable_baseline_fails_open_to_zero():
def test_unresolvable_baseline_remains_unknown():
usage = _usage(fresh=2000, cached=0, written=0, out=500)
assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) == 0.0
def test_a_first_turn_is_the_rate_difference_not_a_switch_penalty():
"""Nothing was cached anywhere on a conversation's first turn, so the baseline would
have paid the same cache write. Charging it to the selected arm alone reported a
fraction of the real saving; on this shape roughly 4% of it.
"""
usage = _usage(fresh=0, cached=0, written=20_000, out=1_000)
first_turn = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False)
opus = litellm.get_model_info("claude-opus-5", "anthropic")
haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic")
both_write = (20_000 * opus["cache_creation_input_token_cost"] + 1_000 * opus["output_cost_per_token"]) - (
20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"]
)
assert first_turn == pytest.approx(both_write)
mid_conversation = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage)
assert first_turn > mid_conversation * 10, "a first turn must not be priced as a switch"
def test_a_first_turn_that_saves_money_never_reports_a_loss():
"""The write premium is fixed by prompt size while the saving grows with completion
length, so charging the write to a first turn made short answers over a large cached
prompt read as losses on requests that genuinely saved. That is the shape most likely
to be on the dashboard, and the sign has to be right.
"""
short_answer = _usage(fresh=0, cached=0, written=20_000, out=200)
assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer, continuing=False) > 0
assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer) < 0
def test_an_undetermined_conversation_shape_stays_conservative():
"""The default must charge the write. A caller that cannot be read, or a surface the
router never classified, has said nothing about whether the baseline was warm, and a
savings figure must not inflate on a guess.
"""
usage = _usage(fresh=0, cached=0, written=20_000, out=1_000)
defaulted = compute_autorouter_savings(
baseline_model="anthropic/claude-opus-5",
selected_model="claude-haiku-4-5",
selected_provider="anthropic",
usage=usage,
)
assert defaulted == pytest.approx(_savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage))
assert defaulted < _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, continuing=False)
def test_a_continuing_turn_on_the_same_model_writes_its_growth_on_both_arms():
"""A conversation that grew by a few tokens writes those on whatever model serves
it, and they are new to every model, so the baseline would have written them too.
Moving them into the baseline's read bucket forgives it a write it really owes and
shrinks the reported saving on ordinary steady-state traffic.
"""
usage = _usage(fresh=0, cached=19_900, written=100, out=1_000)
opus = litellm.get_model_info("claude-opus-5", "anthropic")
haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic")
def cost(info: dict) -> float:
return (
19_900 * info["cache_read_input_token_cost"]
+ 100 * info["cache_creation_input_token_cost"]
+ 1_000 * info["output_cost_per_token"]
)
both_write_the_growth = cost(opus) - cost(haiku)
assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage) == pytest.approx(both_write_the_growth)
def test_a_switch_onto_a_partly_cached_model_still_pays_for_the_write():
"""A model holding a small prefix of this prompt still has to write the rest, and
that write is the switch's cost. Keying the same-model case off reading *anything*
rather than reading *most of it* would hand this request the full rate gap and
inflate the saving by an order of magnitude.
"""
mostly_written = _usage(fresh=0, cached=500, written=19_500, out=1_000)
reported = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", mostly_written)
opus = litellm.get_model_info("claude-opus-5", "anthropic")
haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic")
if_treated_as_same_model = (
500 * opus["cache_read_input_token_cost"]
+ 19_500 * opus["cache_creation_input_token_cost"]
+ 1_000 * opus["output_cost_per_token"]
) - (
500 * haiku["cache_read_input_token_cost"]
+ 19_500 * haiku["cache_creation_input_token_cost"]
+ 1_000 * haiku["output_cost_per_token"]
)
assert reported < if_treated_as_same_model / 10, "a mostly-cold switch must not be priced as a continuation"
assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) is None
def test_a_baseline_that_prices_caching_implicitly_still_pays_for_its_prompt():
@ -794,7 +659,7 @@ def test_a_baseline_that_prices_caching_implicitly_still_pays_for_its_prompt():
selected_model="claude-haiku-4-5",
selected_provider="anthropic",
usage=first_turn,
conversation_continuing=False,
baseline_usage=first_turn,
)
gpt5 = litellm.get_model_info("gpt-5", "openai")
@ -830,7 +695,7 @@ def _priced_chat_model_without_cache_read_rate() -> tuple[str, str, str]:
usage=_usage(fresh=1_000, cached=0, written=0, out=100),
conversation_continuing=True,
)
if priced == 0.0:
if priced is None or priced == 0.0:
continue
return key, key.removeprefix(f"{provider}/"), provider
raise AssertionError("the bundled map has no per-token chat model without a cache-read rate")
@ -848,7 +713,7 @@ def test_a_baseline_with_no_cache_read_rate_is_charged_its_input_rate():
selected_model="claude-haiku-4-5",
selected_provider="anthropic",
usage=continuing,
conversation_continuing=True,
baseline_usage=_usage(0, 20000, 0, 1000),
)
baseline = litellm.get_model_info(baseline_name, baseline_provider)
@ -989,7 +854,7 @@ def test_a_baseline_recorded_on_the_decision_turns_the_driver_on():
compression_saved_tokens=0,
gateway_injected_cache=True,
routing_decision={"conversation_continuing": True, "savings_baseline_model": "anthropic/claude-opus-5"},
usage_object=_cached_usage_object(),
usage_object=_usage(12807, 0, 0, 500).model_dump(),
)
assert result.autorouter != 0.0
@ -1004,13 +869,13 @@ def test_a_leftover_configured_baseline_does_not_override_the_recorded_one(monke
compression_saved_tokens=0,
gateway_injected_cache=True,
routing_decision={"conversation_continuing": True, "savings_baseline_model": "anthropic/claude-opus-5"},
usage_object=_cached_usage_object(),
usage_object=_usage(12807, 0, 0, 500).model_dump(),
)
against_opus = compute_autorouter_savings(
baseline_model="anthropic/claude-opus-5",
selected_model="claude-haiku-4-5",
selected_provider="anthropic",
usage=Usage(**_cached_usage_object()),
usage=_usage(12807, 0, 0, 500),
)
assert result.autorouter == against_opus
@ -1077,12 +942,12 @@ def test_prompt_caching_prices_at_the_deployment_rate_not_the_public_one():
("baseline", "selected", 2.0, None, 0.0, -0.015),
("baseline", "selected", 1.0, None, 0.0, 0.0),
("baseline", "selected", 0.1, 0.004, 0.001, 0.01),
("baseline", "baseline", 0.1, 0.004, 0.001, -0.001),
(None, "selected", 0.1, None, 0.0, 0.0),
("baseline", None, 0.1, None, 0.0, 0.0),
("baseline", "baseline", 0.1, 0.004, 0.001, 0.01),
(None, "selected", 0.1, None, 0.0, 0.006),
("baseline", None, 0.1, None, 0.0, 0.0075),
(None, None, 0.1, None, 0.0, 0.0),
("", "selected", 0.1, None, 0.0, 0.0),
("baseline", "", 0.1, None, 0.0, 0.0),
("", "selected", 0.1, None, 0.0, 0.006),
("baseline", "", 0.1, None, 0.0, 0.0075),
],
)
def test_autorouter_savings_distinguishes_priced_deployments(
@ -1195,7 +1060,7 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate():
compression_saved_tokens=0,
gateway_injected_cache=True,
routing_decision=decision,
usage_object=_cached_usage_object(),
usage_object=_usage(12807, 0, 0, 500).model_dump(),
llm_router=lambda: router,
)
at_public_rate = compute_savings_spend(
@ -1204,7 +1069,7 @@ def test_a_recorded_baseline_deployment_prices_at_its_configured_rate():
compression_saved_tokens=0,
gateway_injected_cache=True,
routing_decision={k: v for k, v in decision.items() if k != "savings_baseline_deployment_id"},
usage_object=_cached_usage_object(),
usage_object=_usage(12807, 0, 0, 500).model_dump(),
llm_router=lambda: router,
)
assert with_deployment_rate.autorouter > at_public_rate.autorouter
@ -1257,9 +1122,8 @@ def test_a_boolean_is_not_a_recorded_savings_figure():
assert result.autorouter == 0.0
def test_rows_written_before_the_field_shipped_recompute():
"""No recorded figure means the row predates the logging-path stamp; the writer
recomputes exactly what the one shared helper would have recorded."""
@pytest.mark.parametrize("continuing", [False, True])
def test_legacy_cache_rows_without_an_estimate_do_not_invent_a_new_figure(continuing: bool) -> None:
from litellm.proxy.spend_tracking.savings import autorouter_savings_for_request
recomputed = compute_savings_spend(
@ -1267,17 +1131,17 @@ def test_rows_written_before_the_field_shipped_recompute():
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=False,
routing_decision=_routed_decision(),
routing_decision={**_routed_decision(), "conversation_continuing": continuing},
usage_object=_cached_usage_object(),
)
direct = autorouter_savings_for_request(
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
routing_decision=_routed_decision(),
routing_decision={**_routed_decision(), "conversation_continuing": continuing},
usage_object=_cached_usage_object(),
)
assert direct is not None and direct != 0.0
assert recomputed.autorouter == direct
assert direct is None
assert recomputed.autorouter == 0.0
def test_driver_off_is_none_not_zero_for_the_request_helper():
@ -1317,7 +1181,7 @@ def test_logging_payload_never_stamps_internal_calls():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
model_id=None,
usage_object=_cached_usage_object(),
usage_object=_usage(12807, 0, 0, 500).model_dump(),
cost_breakdown=None,
)
assert stamped is not None and stamped != 0.0
@ -1327,7 +1191,7 @@ def test_logging_payload_never_stamps_internal_calls():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
model_id=None,
usage_object=_cached_usage_object(),
usage_object=_usage(12807, 0, 0, 500).model_dump(),
cost_breakdown=None,
)
assert internal is None
@ -1343,13 +1207,13 @@ def test_savings_are_net_of_a_priced_classifier():
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
routing_decision=_routed_decision(),
usage_object=_cached_usage_object(),
usage_object=_usage(12807, 0, 0, 500).model_dump(),
)
net = autorouter_savings_for_request(
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
routing_decision={**_routed_decision(), "classifier_cost": 0.005},
usage_object=_cached_usage_object(),
usage_object=_usage(12807, 0, 0, 500).model_dump(),
)
assert gross is not None and net == pytest.approx(gross - 0.005)
@ -1362,13 +1226,13 @@ def test_an_unpriced_classifier_deducts_nothing(classifier_cost: object):
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
routing_decision=_routed_decision(),
usage_object=_cached_usage_object(),
usage_object=_usage(12807, 0, 0, 500).model_dump(),
)
with_cost_field = autorouter_savings_for_request(
model="claude-haiku-4-5",
custom_llm_provider="anthropic",
routing_decision={**_routed_decision(), "classifier_cost": classifier_cost},
usage_object=_cached_usage_object(),
usage_object=_usage(12807, 0, 0, 500).model_dump(),
)
assert with_cost_field == gross
@ -1477,4 +1341,41 @@ def test_marks_gateway_injection_credits_only_the_deployment_that_was_injected()
assert marks_gateway_injection({"litellm_gateway_injected_cache": ""}, "dep-a") is True
assert marks_gateway_injection({"litellm_gateway_injected_cache": ""}, None) is True
assert marks_gateway_injection({"litellm_call_id": "c1"}, "dep-a") is False
@pytest.mark.parametrize("classifier", [0.0, 0.02])
def test_observed_baseline_keeps_both_costs_and_classifier_overhead(classifier: float) -> None:
from litellm.proxy.spend_tracking.savings import BaselineCostSnapshot, price_baseline_comparison
snapshot: Final = BaselineCostSnapshot(
model="baseline", provider="anthropic", prices=None,
actual_spend=0.17, classifier_cost=classifier,
)
restored: Final = BaselineCostSnapshot.model_validate_json(snapshot.model_dump_json())
result: Final = price_baseline_comparison(restored, Usage(prompt_tokens=100, completion_tokens=10), "observed_identical")
assert result is not None
assert result.baseline == snapshot.actual_spend
assert result.actual == snapshot.actual_spend + classifier
assert result.savings == pytest.approx(-classifier)
assert price_baseline_comparison(restored, None, None) is None
def test_modeled_baseline_uses_recorded_prices_and_preserves_other_actual_charges() -> None:
from litellm.proxy.spend_tracking.savings import BaselineCostSnapshot, price_baseline_comparison
snapshot: Final = BaselineCostSnapshot(
model="claude-opus-5", provider="anthropic",
prices={
**litellm.get_model_info("claude-opus-5", custom_llm_provider="anthropic"),
"input_cost_per_token": 0.001, "output_cost_per_token": 0.002,
},
actual_token_cost=0.2, actual_spend=0.23, classifier_cost=0.01,
)
usage: Final = Usage(prompt_tokens=100, completion_tokens=10)
result: Final = price_baseline_comparison(snapshot, usage, "modeled")
assert result is not None
assert result.actual == pytest.approx(0.24)
assert result.baseline == pytest.approx(0.12 + 0.03)
assert result.savings == pytest.approx(-0.09)
assert price_baseline_comparison(snapshot.model_copy(update={"prices": None}), usage, "modeled") is None
assert marks_gateway_injection({"litellm_gateway_injected_cache": True}, "dep-a") is False

6
tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
View file

@ -3745,7 +3745,7 @@ class TestSpendLogsPayload:
"model": "gpt-4o",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
"cache_key": "Cache OFF",
"spend": 0.00022500000000000002,
"total_tokens": 30,
@ -3841,7 +3841,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,
@ -3935,7 +3935,7 @@ class TestSpendLogsPayload:
"model": "claude-4-sonnet-20250514",
"user": "",
"team_id": "",
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
"cache_key": "Cache OFF",
"spend": 0.01383,
"total_tokens": 2598,

15
tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file

@ -3,6 +3,7 @@ import datetime
import json
from collections.abc import Mapping
from datetime import timezone
from types import MappingProxyType
from typing import Any, Final, cast
from unittest.mock import AsyncMock, MagicMock, patch
@ -4650,3 +4651,17 @@ def test_spend_log_request_id_is_the_response_id_a_bridged_messages_caller_recei
)
== "resp_01Lit6806Bridged"
)
def test_baseline_estimate_metadata_comes_from_the_logging_stamp() -> None:
supplied: Final = MappingProxyType({"version": 1, "status": "estimated", "reason": "caller_supplied"})
recorded: Final = MappingProxyType({"version": 1, "status": "unknown", "reason": "history_unavailable"})
result: Final = _get_spend_logs_metadata(
{"autorouter_savings": 999.0, "autorouter_savings_estimate": supplied}, # mutable-ok: legacy metadata helper accepts dicts
autorouter_savings=None,
autorouter_savings_estimate=recorded,
)
assert result["autorouter_savings"] is None
assert result["autorouter_savings_estimate"] == recorded
absent: Final = _get_spend_logs_metadata({"autorouter_savings_estimate": supplied}) # mutable-ok: legacy metadata helper accepts dicts
assert absent["autorouter_savings_estimate"] is None

20
tests/test_litellm/proxy/utils/proxy_logging/test_post_call_failure_hook.py
View file

@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio
from datetime import datetime
from typing import Any, Final
from unittest.mock import AsyncMock, MagicMock
import pytest
@ -13,7 +14,7 @@ from fastapi import HTTPException
import litellm
from litellm.exceptions import GuardrailRaisedException
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import AlertType, ProxyErrorTypes
from litellm.proxy._types import AlertType, ProxyErrorTypes, UserAPIKeyAuth
from litellm.proxy.utils import ProxyLogging
@ -156,6 +157,23 @@ async def test_post_call_failure_hook_non_http_exception_in_callback_swallowed(
assert out is None
@pytest.mark.asyncio
@pytest.mark.parametrize("logging_value", (None, "caller-controlled", {"baseline_cache_context": "untrusted"})) # mutable-ok: emulate an untrusted JSON request field
async def test_terminal_baseline_cleanup_ignores_missing_or_untrusted_logging(
proxy_logging: ProxyLogging, monkeypatch: pytest.MonkeyPatch, logging_value: object
) -> None:
monkeypatch.setattr(litellm, "callbacks", ())
proxy_logging.alert_types = [] # mutable-ok: disable optional alert sinks for this boundary test # rebind-ok: isolate the fixture-owned alert configuration
request_data: Final = {"litellm_call_id": "untrusted-logging", "litellm_logging_obj": logging_value} # mutable-ok: the production failure owner removes internal fields in place
result: Final = await proxy_logging.post_call_failure_hook( # pyright: ignore[reportUnknownMemberType] # exercise the existing proxy terminal owner with its legacy request dictionary contract
request_data=request_data,
original_exception=ValueError("original provider failure"),
user_api_key_dict=UserAPIKeyAuth(request_route="/v1/messages"),
)
assert result is None
assert "litellm_logging_obj" not in request_data
# ---------------------------------------------------------------------------
# _handle_logging_proxy_only_error
# ---------------------------------------------------------------------------

7
tests/test_litellm/repositories/test_repositories.py
View file

@ -2425,6 +2425,10 @@ class TestAutoRouterSessionRepository:
"classifier_cost": 0.01,
"tier_turns": {"complex": 3},
"baseline_models": {"anthropic/claude-opus-5": 3},
"savings_estimated_turns": 3,
"savings_estimated_actual_spend": 0.14,
"savings_estimated_saved_spend": 0.24,
"savings_estimated_baseline_models": {"anthropic/claude-opus-5": 3},
}
@staticmethod
@ -2451,6 +2455,9 @@ class TestAutoRouterSessionRepository:
assert (row.router_name, row.turns, row.spend, row.saved_spend) == ("claude-auto", 3, 0.14, 0.24)
assert row.baseline_models == {"anthropic/claude-opus-5": 3}
assert row.baseline_model == "anthropic/claude-opus-5"
assert row.savings_estimated_turns == 3
assert row.savings_estimated_actual_spend == 0.14
assert row.savings_estimated_saved_spend == 0.24
@pytest.mark.asyncio
async def test_find_latest_for_key_is_none_when_the_key_wrote_no_such_session(self):

13
tests/test_litellm/router_strategy/test_complexity_router.py
View file

@ -8728,19 +8728,26 @@ class TestRecordRoutingDecision:
Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
assert request_kwargs == {}
def test_clearing_the_decision_takes_the_savings_facts_with_it(self):
def test_clearing_the_decision_takes_the_savings_facts_with_it(self) -> None:
"""A fallback to a plain model group re-enters the hook with the same
`request_kwargs`. The baseline and the conversation shape ride inside the
decision rather than beside it, so one clear cannot leave either behind and
attribute an auto-router saving to a deployment that never routed."""
decision = {
from litellm.types.router import BaselineRouteStamp
decision: Final = {
"router_model_name": "smart-router",
"router_type": "complexity",
"routed_model": "gpt-4o-mini",
"savings_baseline_model": "anthropic/claude-opus-5",
"savings_baseline_deployment_id": "opus-deployment",
"conversation_continuing": False,
}
request_kwargs: Dict = {"litellm_metadata": {"routing_decision": decision}}
request_kwargs: Final[dict[str, dict[str, object]]] = {"litellm_metadata": {}}
Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=decision)
stamp: Final = request_kwargs["litellm_metadata"]["_autorouter_baseline_route"]
assert isinstance(stamp, BaselineRouteStamp)
assert stamp.baseline_deployment_id == "opus-deployment"
Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
assert request_kwargs["litellm_metadata"] == {}

1
tests/test_litellm/test_utils.py
View file

@ -869,6 +869,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"supports_pdf_input": {"type": "boolean"},
"prompt_cache_min_tokens": {"type": "number"},
"supports_prompt_cache_breakpoint": {"type": "boolean"},
"supports_thinking_cache_preservation": {"type": "boolean"},
"supports_prompt_caching": {"type": "boolean"},
"supports_response_schema": {"type": "boolean"},
"supports_system_messages": {"type": "boolean"},

37
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx
View file

@ -68,6 +68,8 @@ const totals = (overrides: Partial<Totals> = {}): Totals => ({
avg_session_seconds: 7560,
avg_tokens_per_session: 5_300_000,
spend: 359.86,
savings_estimated_turns: overrides.turns ?? 3073,
savings_estimated_actual_spend: overrides.spend ?? 359.86,
classifier_cost: 6.146,
saved_spend: 2174.59,
baseline_spend: 2534.45,
@ -100,6 +102,8 @@ const zeroTotals: Totals = {
avg_session_seconds: 0,
avg_tokens_per_session: 0,
spend: 0,
savings_estimated_turns: 0,
savings_estimated_actual_spend: 0,
classifier_cost: 0,
saved_spend: 0,
baseline_spend: 0,
@ -153,6 +157,39 @@ describe("AutoRouterBenchmarksTab", () => {
mockAutoRouters();
});
it.each([
{ estimatedTurns: 0, saved: null, pct: null },
{ estimatedTurns: 10, saved: -0.5, pct: -33.3 },
{ estimatedTurns: 10, saved: 0, pct: 0 },
])("preserves costs for $estimatedTurns estimated turns with savings $saved", ({ estimatedTurns, saved, pct }) => {
const cohort = {
savings_estimated_turns: estimatedTurns,
savings_estimated_actual_spend: estimatedTurns ? 2 : 0,
saved_spend: saved,
baseline_spend: estimatedTurns ? 2 + (saved ?? 0) : null,
saved_pct: pct,
saved_per_session: null,
};
const partial = totals(cohort);
mockHook({ data: response([], partial) });
renderTab();
expect(screen.getByText("Estimated savings on covered turns")).toBeInTheDocument();
expect(screen.getByText(`${estimatedTurns} of 3,073 turns estimated`)).toBeInTheDocument();
expect(screen.getByText("$359.86")).toBeInTheDocument();
expect(screen.getByText("Actual spend on covered turns")).toBeInTheDocument();
expect(screen.getByText("Estimated baseline spend on covered turns")).toBeInTheDocument();
expect(screen.getAllByText("Unavailable")).toHaveLength(estimatedTurns ? 1 : 3);
if (saved === 0) {
expect(screen.getByText("0%")).toBeInTheDocument();
expect(screen.getAllByText("$2.00")).toHaveLength(2);
} else if (estimatedTurns) {
expect(screen.getByText("-$0.5000")).toBeInTheDocument();
expect(screen.getByText("+33%")).toBeInTheDocument();
} else {
expect(screen.queryByText("+0%")).not.toBeInTheDocument();
}
});
it("leads with total estimated savings, before the four session-shape metrics", () => {
mockHook({ data: response([group(), group({ router_name: "gpt-auto" })]) });
renderTab();

56
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx
View file

@ -73,26 +73,37 @@ const SpendRow: React.FC<{ label: string; value: string; hint?: string; subdued?
const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => {
const stats = view.stats;
const cheaper = stats.saved_spend >= 0;
const cheaper = stats.saved_spend != null && stats.saved_spend >= 0;
const completeCoverage = stats.savings_estimated_turns === stats.turns;
return (
<Card className="overflow-hidden py-0">
<div className="grid md:grid-cols-[minmax(0,1fr)_minmax(0,1fr)]">
<div className="flex flex-col items-center justify-center gap-2 p-6">
<p className="text-xs font-semibold uppercase tracking-wider text-muted-foreground">
Total estimated savings
{completeCoverage ? "Total estimated savings" : "Estimated savings on covered turns"}
</p>
<div className="flex flex-wrap items-center justify-center gap-3">
<p className="min-w-0 break-all text-center text-4xl font-semibold tracking-tight text-foreground xl:text-6xl">
{usd(stats.saved_spend)}
{stats.saved_spend == null ? "Unavailable" : usd(stats.saved_spend)}
</p>
<Badge
variant="secondary"
className={`h-6 px-2.5 text-sm ${cheaper ? "bg-success/10 text-success" : "bg-destructive/10 text-destructive"}`}
>
{stats.saved_spend !== 0 && (cheaper ? "-" : "+")}
{Math.abs(stats.saved_pct).toFixed(0)}%
</Badge>
{stats.saved_pct != null && (
<Badge
variant="secondary"
className={`h-6 px-2.5 text-sm ${cheaper ? "bg-success/10 text-success" : "bg-destructive/10 text-destructive"}`}
>
{stats.saved_spend !== 0 && (cheaper ? "-" : "+")}
{Math.abs(stats.saved_pct).toFixed(0)}%
</Badge>
)}
</div>
<p className="text-center text-xs text-muted-foreground">
{stats.savings_estimated_turns.toLocaleString()} of {stats.turns.toLocaleString()} turns estimated
</p>
{!completeCoverage && (
<p className="text-center text-xs text-muted-foreground">
Turns without a current estimate are excluded, including older estimates.
</p>
)}
</div>
<div className="flex flex-col justify-center border-t p-6 md:border-t-0 md:border-l">
@ -120,7 +131,15 @@ const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => {
</p>
)}
<Separator />
<SpendRow label="Estimated spend at highest-tier model" value={usd(stats.baseline_spend)} />
{!completeCoverage && (
<SpendRow label="Actual spend on covered turns" value={usd(stats.savings_estimated_actual_spend)} />
)}
<SpendRow
label={
completeCoverage ? "Estimated spend at highest-tier model" : "Estimated baseline spend on covered turns"
}
value={stats.baseline_spend == null ? "Unavailable" : usd(stats.baseline_spend)}
/>
</div>
</div>
</Card>
@ -279,7 +298,7 @@ const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data,
<div className="grid grid-cols-1 gap-4 sm:grid-cols-2 lg:grid-cols-4">
<Metric
label="Avg saved per session"
value={usd(stats.saved_per_session)}
value={stats.saved_per_session == null ? "Unavailable" : usd(stats.saved_per_session)}
hint={`· ${stats.sessions.toLocaleString()} sessions`}
/>
<Metric label="Avg turns per session" value={stats.avg_turns_per_session.toFixed(1)} />
@ -288,12 +307,13 @@ const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data,
</div>
<p className="text-xs text-muted-foreground">
Compares your actual routed spend with the estimated cost of using only the most expensive model configured in
the auto-router. It accounts for both the cache savings from staying on one model and the added cache costs from
switching models. Savings are net of recorded LLM classification cost, which is included in actual spend.
Classification cost per 1K turns is averaged over all auto-router turns, including those that skip
classification. The range counts whole sessions that overlap it, so totals can differ slightly from the Overall
tab, which buckets savings by UTC day.
Compares covered turns with the estimated cost of using the router&apos;s highest-tier baseline model. Estimates
use registered requests since tracking began, matching cache prefixes and expiry, and the actual response
length. Total actual spend includes every turn; savings and baseline spend include only turns with a current
estimate, including turns with zero savings. Savings are net of recorded LLM classification cost. Classification
cost per 1K turns is averaged over all auto-router turns, including those that skip classification. The range
counts whole sessions that overlap it, so totals can differ slightly from the Overall tab, which buckets savings
by UTC day.
</p>
<div className="space-y-4">

2
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/TierTurnsChart.test.tsx
View file

@ -21,6 +21,8 @@ const totalsOnly = {
avg_session_seconds: 60,
avg_tokens_per_session: 100,
spend: 1,
savings_estimated_turns: 9,
savings_estimated_actual_spend: 1,
saved_spend: 1,
baseline_spend: 2,
saved_pct: 50,

2
ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.test.ts
View file

@ -37,6 +37,8 @@ const totals = (overrides: Partial<AutoRouterBenchmarkGroup> = {}) => ({
avg_session_seconds: 7560,
avg_tokens_per_session: 5_300_000,
spend: 359.86,
savings_estimated_turns: overrides.turns ?? 3073,
savings_estimated_actual_spend: overrides.spend ?? 359.86,
classifier_cost: 6.146,
saved_spend: 2174.59,
baseline_spend: 2534.45,

8
ui/litellm-dashboard/src/components/shared/SavingsTiles.tsx
View file

@ -37,10 +37,10 @@ const SavingsTiles = ({ results, isLoading }: { results: DailyData[]; isLoading:
return (
<div className="grid grid-cols-1 gap-6 sm:grid-cols-2 lg:grid-cols-4">
<SummaryCard
label="Total saved"
label="Total recorded savings"
value={usd(totals.total)}
hint={isLoading ? "Loading..." : "Compression + prompt caching + auto-router"}
info="The sum of the three tiles beside it. Its caching term is the LiteLLM-injected share, so this total is what the gateway itself delivered; caching that clients or providers brought on their own appears only in the caching tile's Total figure."
info="The sum of recorded savings in the three tiles beside it. Auto-router requests without an estimate are excluded. Its caching term is the LiteLLM-injected share; caching supplied by clients or providers appears only in the caching tile's Total figure."
/>
<SummaryCard
label="Compression savings"
@ -58,8 +58,8 @@ const SavingsTiles = ({ results, isLoading }: { results: DailyData[]; isLoading:
<SummaryCard
label="Auto-router savings"
value={usd(totals.autorouter)}
hint="vs. the priciest model it could pick"
info="What this traffic would have cost had every request gone to the most expensive model the auto-router can route to, minus what it actually cost. Switching leaves the new model with a cold cache, so it pays to write the prompt again while the baseline is priced as already warm; a route that thrashes the cache can total below zero, and a genuine first turn, where neither side had anything cached, is undercounted."
hint="Recorded estimates subtotal"
info="Sum of available per-request savings estimates against each router's highest-tier baseline, net of classifier cost. Requests without an estimate contribute nothing to this subtotal; this does not mean they saved zero. Historical records retain the estimator used when they were written. The Auto-router usage tab shows coverage for current estimates."
/>
</div>
);

2
ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx
View file

@ -32,6 +32,8 @@ const stats = {
avg_session_seconds: 30,
avg_tokens_per_session: 100,
spend: 1.25,
savings_estimated_turns: 4,
savings_estimated_actual_spend: 1.25,
classifier_cost: 0.25,
saved_spend: 8.75,
baseline_spend: 10,

3
ui/litellm-dashboard/src/components/templates/KeySavingsTab.integration.test.tsx
View file

@ -94,7 +94,7 @@ describe("KeySavingsTab", () => {
renderTab();
expect(screen.getByTestId("summary-card-total-saved")).toHaveTextContent("$5.40");
expect(screen.getByTestId("summary-card-total-recorded-savings")).toHaveTextContent("$5.40");
expect(screen.getByTestId("summary-card-compression-savings")).toHaveTextContent("$2.00");
expect(screen.getByTestId("summary-card-compression-savings")).toHaveTextContent("1,000 tokens compressed");
// the card leads with what LiteLLM's own injection earned and carries the total beneath it,
@ -102,6 +102,7 @@ describe("KeySavingsTab", () => {
expect(screen.getByTestId("summary-card-prompt-caching-savings")).toHaveTextContent("$0.40");
expect(screen.getByTestId("summary-card-prompt-caching-savings")).toHaveTextContent("$1.00Total");
expect(screen.getByTestId("summary-card-auto-router-savings")).toHaveTextContent("$3.00");
expect(screen.getByTestId("summary-card-auto-router-savings")).toHaveTextContent("Recorded estimates subtotal");
});
it("separates a key with no traffic from one still loading", () => {

85
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -23556,9 +23556,9 @@ export interface components {
avg_turns_per_session: number;
/**
* Baseline Spend
* @description spend plus saved_spend: the estimated single-model cost
* @description Estimated single-model cost for covered turns only
*/
baseline_spend: number;
baseline_spend: number | null;
cache: components["schemas"]["AutoRouterCacheStats"];
/**
* Classifier Cost
@ -23577,16 +23577,29 @@ export interface components {
router_type: string;
/**
* Saved Pct
* @description saved_spend over baseline_spend, as a percentage
* @description Covered savings over covered baseline spend, as a percentage
*/
saved_pct: number;
/** Saved Per Session */
saved_per_session: number;
saved_pct: number | null;
/**
* Saved Per Session
* @description Average session savings; unavailable unless every turn is covered
*/
saved_per_session: number | null;
/**
* Saved Spend
* @description Signed dollars saved versus each router's savings baseline (derived from its hardest tier, or the configured override), from the same per-request savings record the usage tab reads
* @description Signed savings for covered turns only; null when traffic has no current estimates
*/
saved_spend: number;
saved_spend: number | null;
/**
* Savings Estimated Actual Spend
* @description Actual spend, including classifier cost, for covered turns only
*/
savings_estimated_actual_spend: number;
/**
* Savings Estimated Turns
* @description Turns covered by the current savings estimator; legacy estimates are excluded
*/
savings_estimated_turns: number;
/** Sessions */
sessions: number;
/**
@ -23617,9 +23630,9 @@ export interface components {
avg_turns_per_session: number;
/**
* Baseline Spend
* @description spend plus saved_spend: the estimated single-model cost
* @description Estimated single-model cost for covered turns only
*/
baseline_spend: number;
baseline_spend: number | null;
cache: components["schemas"]["AutoRouterCacheStats"];
/**
* Classifier Cost
@ -23628,16 +23641,29 @@ export interface components {
classifier_cost: number | null;
/**
* Saved Pct
* @description saved_spend over baseline_spend, as a percentage
* @description Covered savings over covered baseline spend, as a percentage
*/
saved_pct: number;
/** Saved Per Session */
saved_per_session: number;
saved_pct: number | null;
/**
* Saved Per Session
* @description Average session savings; unavailable unless every turn is covered
*/
saved_per_session: number | null;
/**
* Saved Spend
* @description Signed dollars saved versus each router's savings baseline (derived from its hardest tier, or the configured override), from the same per-request savings record the usage tab reads
* @description Signed savings for covered turns only; null when traffic has no current estimates
*/
saved_spend: number;
saved_spend: number | null;
/**
* Savings Estimated Actual Spend
* @description Actual spend, including classifier cost, for covered turns only
*/
savings_estimated_actual_spend: number;
/**
* Savings Estimated Turns
* @description Turns covered by the current savings estimator; legacy estimates are excluded
*/
savings_estimated_turns: number;
/** Sessions */
sessions: number;
/**
@ -23907,21 +23933,21 @@ export interface components {
AutoRouterSessionResponse: {
/**
* Baseline Model
* @description The savings baseline most of this session's turns were priced against, recorded turn by turn, so it still names the counterfactual after the router is reconfigured or removed. None when no turn recorded one: rows from before the baseline was recorded, and adaptive and quality routers, which derive no baseline and so report no savings
* @description The savings baseline most covered turns were priced against, recorded turn by turn, so it still names the counterfactual after the router is reconfigured or removed. None when no turn recorded one: rows from before the baseline was recorded, and adaptive and quality routers, which derive no baseline and so report no savings
*/
baseline_model: string | null;
/**
* Baseline Models
* @description Turns priced against each baseline model; more than one entry means the router's baseline changed mid-session and baseline_spend mixes both
* @description Covered turns priced against each baseline model; more than one entry means the router's baseline changed mid-session and baseline_spend mixes both
*/
baseline_models: {
[key: string]: number;
};
/**
* Baseline Spend
* @description spend plus saved_spend: the estimated single-model cost
* @description Estimated single-model cost; unavailable unless every turn is covered
*/
baseline_spend: number;
baseline_spend: number | null;
/**
* Last Model
* @description The deployment model the most recent turn was routed to
@ -23939,9 +23965,24 @@ export interface components {
router_type: string;
/**
* Saved Spend
* @description Estimated savings against the baseline, net of classifier cost
* @description Estimated savings for covered turns only, net of classifier cost
*/
saved_spend: number;
saved_spend: number | null;
/**
* Savings Estimated Actual Spend
* @description Actual spend, including classifier cost, for covered turns only
*/
savings_estimated_actual_spend: number;
/**
* Savings Estimated Baseline Spend
* @description Estimated single-model cost for covered turns only
*/
savings_estimated_baseline_spend: number | null;
/**
* Savings Estimated Turns
* @description Turns covered by the current savings estimator; legacy estimates are excluded
*/
savings_estimated_turns: number;
/** Session Id */
session_id: string;
/**