fix(autorouter): preserve historical savings and align dashboard totals

This commit is contained in:
Tin Chi Lo 2026-09-22 18:56:17 -07:00
parent 5fa1257b7c
commit b0b24e7e5c
41 changed files with 2872 additions and 288 deletions

View file

@ -0,0 +1,172 @@
BEGIN;
SET LOCAL lock_timeout = '10s';
SET LOCAL statement_timeout = '30s';
DO $install$
DECLARE
target_schema CONSTANT TEXT := current_schema();
daily_table CONSTANT REGCLASS := to_regclass(format('%I.%I', target_schema, 'LiteLLM_DailyUserSpend'));
observation_table CONSTANT REGCLASS := to_regclass(format('%I.%I', target_schema, 'LiteLLM_AutoRouterBaselineObservation'));
-- This marker versions the full column/function/trigger contract, preserved by older pods.
implementation_version CONSTANT INTEGER := 1;
function_body CONSTANT TEXT := $function$
DECLARE
previous_publication JSONB := OLD.daily_costs_publication::jsonb;
next_publication JSONB := NEW.publication::jsonb;
attribution JSONB := NEW.data::jsonb -> 'daily';
previous_estimated BOOLEAN := COALESCE(previous_publication ->> 'status' = 'estimated', FALSE)
AND previous_publication ->> 'actual_spend' IS NOT NULL
AND previous_publication ->> 'baseline_spend' IS NOT NULL;
next_estimated BOOLEAN := COALESCE(next_publication ->> 'status' = 'estimated', FALSE)
AND next_publication ->> 'actual_spend' IS NOT NULL
AND next_publication ->> 'baseline_spend' IS NOT NULL;
request_delta BIGINT := (CASE WHEN next_estimated THEN 1 ELSE 0 END)
- (CASE WHEN previous_estimated THEN 1 ELSE 0 END);
actual_delta DOUBLE PRECISION :=
(CASE WHEN next_estimated THEN (next_publication ->> 'actual_spend')::double precision ELSE 0 END)
- (CASE WHEN previous_estimated THEN (previous_publication ->> 'actual_spend')::double precision ELSE 0 END);
target_user_id TEXT;
BEGIN
NEW.daily_costs_publication := NEW.publication;
IF request_delta = 0 AND actual_delta = 0 THEN
RETURN NEW;
END IF;
FOR target_user_id IN
SELECT DISTINCT COALESCE(target ->> 'entity_id', '')
FROM jsonb_array_elements(COALESCE(attribution -> 'targets', '[]'::jsonb)) AS target
WHERE target ->> 'entity' = 'user'
ORDER BY 1
LOOP
INSERT INTO "LiteLLM_DailyUserSpend" (
id, user_id, date, api_key, model, custom_llm_provider,
mcp_namespaced_tool_name, endpoint, model_group,
autorouter_estimated_requests, autorouter_estimated_actual_spend, updated_at
) VALUES (
'autorouter-coverage:' || length(NEW.request_id)::text || ':' || NEW.request_id || ':' || target_user_id,
target_user_id, COALESCE(attribution ->> 'date', ''), COALESCE(attribution ->> 'api_key', ''),
COALESCE(attribution ->> 'model', ''), COALESCE(attribution ->> 'custom_llm_provider', ''),
COALESCE(attribution ->> 'mcp_namespaced_tool_name', ''), COALESCE(attribution ->> 'endpoint', ''),
attribution ->> 'model_group', request_delta, actual_delta, (NOW() AT TIME ZONE 'UTC')
)
ON CONFLICT (user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint)
DO UPDATE SET
autorouter_estimated_requests = "LiteLLM_DailyUserSpend".autorouter_estimated_requests
+ EXCLUDED.autorouter_estimated_requests,
autorouter_estimated_actual_spend = "LiteLLM_DailyUserSpend".autorouter_estimated_actual_spend
+ EXCLUDED.autorouter_estimated_actual_spend,
updated_at = (NOW() AT TIME ZONE 'UTC');
END LOOP;
RETURN NEW;
END;
$function$;
missing_columns BOOLEAN;
invalid_columns BOOLEAN;
function_id OID;
function_version INTEGER;
function_ready BOOLEAN;
trigger_ready BOOLEAN;
BEGIN
IF daily_table IS NULL OR observation_table IS NULL THEN
RAISE EXCEPTION 'Daily auto-router coverage requires its tables in schema %', target_schema;
END IF;
FOR attempt IN 1..2 LOOP
SELECT bool_or(a.attnum IS NULL),
bool_or(a.attnum IS NOT NULL AND (
a.atttypid <> required.column_type::regtype
OR a.attnotnull <> required.not_null
OR CASE WHEN required.not_null
THEN COALESCE(pg_get_expr(d.adbin, d.adrelid), '') !~ '^0([.]0+)?(::(bigint|double precision))?$'
ELSE d.oid IS NOT NULL END
))
INTO missing_columns, invalid_columns
FROM (VALUES
(daily_table, 'autorouter_accounted_requests', 'bigint', TRUE),
(daily_table, 'autorouter_requests', 'bigint', TRUE),
(daily_table, 'autorouter_llm_spend', 'double precision', TRUE),
(daily_table, 'autorouter_classifier_cost', 'double precision', TRUE),
(daily_table, 'autorouter_classifier_cost_recorded_requests', 'bigint', TRUE),
(daily_table, 'autorouter_estimated_requests', 'bigint', TRUE),
(daily_table, 'autorouter_estimated_actual_spend', 'double precision', TRUE),
(observation_table, 'daily_costs_publication', 'text', FALSE)
) AS required(table_id, column_name, column_type, not_null)
LEFT JOIN pg_attribute a ON a.attrelid = required.table_id
AND a.attname = required.column_name AND a.attnum > 0 AND NOT a.attisdropped
LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum;
IF invalid_columns THEN
RAISE EXCEPTION 'Daily auto-router coverage has incompatible column definitions in schema %', target_schema;
END IF;
SELECT p.oid,
substring(obj_description(p.oid, 'pg_proc') FROM '^litellm:autorouter_daily_coverage:([0-9]{1,9})$')::integer,
p.prorettype = 'trigger'::regtype AND l.lanname = 'plpgsql'
AND p.provolatile = 'v' AND NOT p.prosecdef AND p.proconfig IS NULL
AND (p.prosrc = function_body OR COALESCE(
substring(obj_description(p.oid, 'pg_proc') FROM '^litellm:autorouter_daily_coverage:([0-9]{1,9})$')::integer,
0
) > implementation_version)
INTO function_id, function_version, function_ready
FROM pg_proc p
JOIN pg_namespace n ON n.oid = p.pronamespace
JOIN pg_language l ON l.oid = p.prolang
WHERE n.nspname = target_schema AND p.proname = 'litellm_update_daily_autorouter_coverage'
AND p.pronargs = 0;
SELECT EXISTS (
SELECT 1 FROM pg_trigger t
JOIN pg_attribute a ON a.attrelid = observation_table AND a.attname = 'publication'
AND a.attnum > 0 AND NOT a.attisdropped
WHERE t.tgrelid = observation_table AND t.tgname = 'litellm_update_daily_autorouter_coverage'
AND NOT t.tgisinternal AND t.tgenabled IN ('O', 'A') AND t.tgtype = 19
AND t.tgattr::text = a.attnum::text AND t.tgnargs = 0 AND t.tgqual IS NULL
AND t.tgfoid = function_id
) INTO trigger_ready;
IF NOT missing_columns AND COALESCE(function_ready, FALSE) AND trigger_ready THEN
RETURN;
END IF;
IF function_version > implementation_version THEN
RAISE EXCEPTION 'Newer daily auto-router coverage is incompatible with this build in schema %', target_schema;
END IF;
IF attempt = 1 THEN
PERFORM pg_advisory_xact_lock(1279874117, 1145129292);
END IF;
END LOOP;
IF missing_columns THEN
ALTER TABLE "LiteLLM_DailyUserSpend"
ADD COLUMN IF NOT EXISTS "autorouter_accounted_requests" BIGINT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS "autorouter_requests" BIGINT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS "autorouter_llm_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS "autorouter_classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS "autorouter_classifier_cost_recorded_requests" BIGINT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS "autorouter_estimated_requests" BIGINT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS "autorouter_estimated_actual_spend" DOUBLE PRECISION NOT NULL DEFAULT 0;
ALTER TABLE "LiteLLM_AutoRouterBaselineObservation"
ADD COLUMN IF NOT EXISTS "daily_costs_publication" TEXT;
END IF;
IF NOT COALESCE(function_ready, FALSE) THEN
EXECUTE format(
'CREATE OR REPLACE FUNCTION %I.litellm_update_daily_autorouter_coverage() '
'RETURNS TRIGGER AS %L LANGUAGE plpgsql VOLATILE SECURITY INVOKER',
target_schema, function_body
);
EXECUTE format('ALTER FUNCTION %I.litellm_update_daily_autorouter_coverage() RESET ALL', target_schema);
EXECUTE format('COMMENT ON FUNCTION %I.litellm_update_daily_autorouter_coverage() IS %L',
target_schema, 'litellm:autorouter_daily_coverage:' || implementation_version::text);
END IF;
IF NOT trigger_ready THEN
DROP TRIGGER IF EXISTS litellm_update_daily_autorouter_coverage ON "LiteLLM_AutoRouterBaselineObservation";
CREATE TRIGGER litellm_update_daily_autorouter_coverage
BEFORE UPDATE OF publication ON "LiteLLM_AutoRouterBaselineObservation"
FOR EACH ROW EXECUTE PROCEDURE litellm_update_daily_autorouter_coverage();
END IF;
END;
$install$;
COMMIT;

View file

@ -805,6 +805,13 @@ model LiteLLM_DailyUserSpend {
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
autorouter_accounted_requests BigInt @default(0)
autorouter_requests BigInt @default(0)
autorouter_llm_spend Float @default(0.0)
autorouter_classifier_cost Float @default(0.0)
autorouter_classifier_cost_recorded_requests BigInt @default(0)
autorouter_estimated_requests BigInt @default(0)
autorouter_estimated_actual_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -1579,6 +1586,7 @@ model LiteLLM_AutoRouterBaselineObservation {
revision BigInt
data String
publication String?
daily_costs_publication String?
conflicted Boolean @default(false)
@@index([scope, started_at, request_id], map: "idx_autorouter_baseline_event_order")

View file

@ -1145,6 +1145,38 @@ class ProxyExtrasDBManager:
prisma_env=prisma_env,
)
@staticmethod
def apply_autorouter_daily_coverage() -> None:
from litellm_proxy_extras.migration_lock import migration_environment
package_dir: Final = Path(__file__).resolve().parent
migration: Final = package_dir / "migrations" / "20260923000000_add_daily_autorouter_costs" / "migration.sql"
for attempt in range(MAX_MIGRATE_DEPLOY_ATTEMPTS):
try:
prisma_toolchain.run_prisma(
[
_get_prisma_command(),
"db",
"execute",
"--schema",
str(package_dir / "schema.prisma"),
"--file",
str(migration),
],
timeout=prisma_command_timeout(),
env=migration_environment(_get_prisma_env()),
stdout=None,
stderr=None,
)
return
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
if isinstance(error, OSError) or attempt + 1 == MAX_MIGRATE_DEPLOY_ATTEMPTS:
raise RuntimeError(
"Failed to install daily auto-router coverage; database setup cannot continue"
) from error
logger.warning("Daily auto-router coverage installation attempt %s failed; retrying", attempt + 1)
time.sleep(random.uniform(0.5, 1.5))
@staticmethod
def setup_database(
use_migrate: bool = False, use_v2_resolver: bool = False
@ -1170,6 +1202,7 @@ class ProxyExtrasDBManager:
use_migrate=use_migrate, use_v2_resolver=use_v2_resolver
)
if migrated:
ProxyExtrasDBManager.apply_autorouter_daily_coverage()
ProxyExtrasDBManager.repair_invalid_indexes()
ProxyExtrasDBManager.apply_replica_identity_full_if_requested()
return migrated

View file

@ -34,15 +34,12 @@ class LiteLLM_AutoRouterSession(LiteLLMPydanticObjectBase):
@property
def baseline_model(self) -> str | None:
"""The baseline most covered turns were priced against, or None when none were estimated.
"""The baseline recorded by most session turns, including historical turns.
A router reconfigured mid-session leaves turns priced against two baselines; the row keeps both
counts, and the label is the one that priced the most money-carrying turns rather than whatever the
counts, and the label is the one recorded by the most turns rather than whatever the
router is configured with now.
"""
if not self.savings_estimated_baseline_models:
if not self.baseline_models:
return None
return max(
self.savings_estimated_baseline_models,
key=lambda model: (self.savings_estimated_baseline_models[model], model),
)
return max(self.baseline_models, key=lambda model: (self.baseline_models[model], model))

View file

@ -5484,6 +5484,13 @@ class BaseDailySpendTransaction(TypedDict):
# missing value to zero, so requiring it here would describe a shape the aggregation
# is explicitly tested against.
autorouter_savings_spend: NotRequired[float]
autorouter_accounted_requests: ReadOnly[NotRequired[int]]
autorouter_requests: ReadOnly[NotRequired[int]]
autorouter_llm_spend: ReadOnly[NotRequired[float]]
autorouter_classifier_cost: ReadOnly[NotRequired[float]]
autorouter_classifier_cost_recorded_requests: ReadOnly[NotRequired[int]]
autorouter_estimated_requests: ReadOnly[NotRequired[int]]
autorouter_estimated_actual_spend: ReadOnly[NotRequired[float]]
# request level metrics
spend: float

View file

@ -70,6 +70,7 @@ class Session(NamedTuple):
turns: int | None = None
savings_estimated_turns: int | None = None
savings_estimated_actual_spend: float | None = None
savings_estimated_baseline_spend: float | None = None
class Credentials(NamedTuple):
@ -210,7 +211,8 @@ def _session_from_payload(payload: Mapping[str, object]) -> Session | None:
router_name: Final = printable(payload.get("router_name"))
last_model: Final = printable(payload.get("last_model"))
spend: Final = payload.get("spend")
baseline_spend: Final = payload.get("savings_estimated_baseline_spend", payload.get("baseline_spend"))
baseline_spend: Final = payload.get("baseline_spend")
estimated_baseline_spend: Final = payload.get("savings_estimated_baseline_spend")
turns: Final = payload.get("turns")
estimated_turns: Final = payload.get("savings_estimated_turns")
estimated_actual: Final = payload.get("savings_estimated_actual_spend")
@ -222,6 +224,12 @@ def _session_from_payload(payload: Mapping[str, object]) -> Session | None:
not isinstance(baseline_spend, (int, float)) or isinstance(baseline_spend, bool) or not isfinite(baseline_spend)
):
return None
if estimated_baseline_spend is not None and (
not isinstance(estimated_baseline_spend, (int, float))
or isinstance(estimated_baseline_spend, bool)
or not isfinite(estimated_baseline_spend)
):
return None
return Session(
router_name=router_name,
last_model=last_model,
@ -242,11 +250,14 @@ def _session_from_payload(payload: Mapping[str, object]) -> Session | None:
and estimated_actual >= 0
else None
),
savings_estimated_baseline_spend=(
float(estimated_baseline_spend) if estimated_baseline_spend is not None else None
),
)
def cache_path(cache_dir: Path, credentials: Credentials, session_id: str) -> Path:
identity: Final = "\n".join((credentials.base_url, credentials.api_key, session_id))
identity: Final = "\n".join(("v2", credentials.base_url, credentials.api_key, session_id))
return cache_dir / hashlib.sha256(identity.encode()).hexdigest()
@ -342,35 +353,41 @@ def render(model: str, session: Session | None, config_dir: Path, use_color: boo
routed: Final = paint(BOLD, f"Routed to: {model}")
if session is None:
return routed
if session.savings_estimated_turns == 0 or session.baseline_spend is None:
covered_only: Final = session.baseline_spend is None
baseline_spend: Final = session.savings_estimated_baseline_spend if covered_only else session.baseline_spend
if baseline_spend is None or (covered_only and session.savings_estimated_turns == 0):
return f"{routed}{SEPARATOR}Savings unavailable"
if session.baseline_model is None or session.baseline_spend <= 0:
if session.baseline_model is None or baseline_spend <= 0:
return routed
if session.savings_estimated_turns is not None and (
session.savings_estimated_actual_spend is None
or session.turns is None
or session.savings_estimated_turns > session.turns
if (
covered_only
and session.savings_estimated_turns is not None
and (
session.savings_estimated_actual_spend is None
or session.turns is None
or session.savings_estimated_turns > session.turns
)
):
return f"{routed}{SEPARATOR}Savings unavailable"
compared_spend: Final = (
session.savings_estimated_actual_spend
if session.savings_estimated_turns is not None and session.savings_estimated_actual_spend is not None
if covered_only and session.savings_estimated_actual_spend is not None
else session.spend
)
coverage: Final = (
f"{SEPARATOR}{session.savings_estimated_turns} of {session.turns} turns estimated"
if session.savings_estimated_turns is not None
if covered_only and session.savings_estimated_turns is not None
else ""
)
reference: Final = baseline_label(session.baseline_model, config_dir)
pct: Final = round((session.baseline_spend - compared_spend) / session.baseline_spend * 100)
pct: Final = round((baseline_spend - compared_spend) / baseline_spend * 100)
sign: Final = "-" if pct > 0 else "+" if pct < 0 else ""
delta: Final = paint(LITELLM_COLOR, f"{sign}{abs(pct)}% vs {reference}")
peak: Final = max(compared_spend, session.baseline_spend)
peak: Final = max(compared_spend, baseline_spend)
label_width: Final = max(_display_width(session.router_name), _display_width(reference))
rows: Final = (
(session.router_name, compared_spend, LITELLM_COLOR),
(reference, session.baseline_spend, BASELINE_COLOR),
(reference, baseline_spend, BASELINE_COLOR),
)
lines: Final = (
f"{paint(DIM, label + ' ' * (label_width - _display_width(label)))} "

View file

@ -0,0 +1,75 @@
from math import isclose
from typing import Final, Literal
from pydantic import BaseModel, ConfigDict
DAILY_COSTS_COMPLETE_SQL: Final = """
autorouter_accounted_requests = api_requests
AND autorouter_accounted_requests >= 0
AND autorouter_requests BETWEEN 0 AND successful_requests
AND autorouter_classifier_cost_recorded_requests BETWEEN 0 AND autorouter_requests
AND autorouter_estimated_requests BETWEEN 0 AND autorouter_requests
AND (autorouter_savings_spend = 0 OR autorouter_estimated_requests > 0)
"""
AUTOROUTER_DAILY_COSTS_SQL: Final = f"""
SELECT
COALESCE(SUM(autorouter_requests), 0)::bigint AS requests,
COALESCE(SUM(autorouter_llm_spend), 0)::float8 AS llm_spend,
COALESCE(SUM(autorouter_classifier_cost), 0)::float8 AS classifier_cost,
COALESCE(SUM(autorouter_classifier_cost_recorded_requests), 0)::bigint AS classifier_requests,
COALESCE(SUM(autorouter_estimated_requests), 0)::bigint AS estimated_requests,
COALESCE(SUM(autorouter_estimated_actual_spend), 0)::float8 AS estimated_actual_spend,
COALESCE(SUM(autorouter_savings_spend), 0)::float8 AS saved_spend,
COALESCE(BOOL_AND({DAILY_COSTS_COMPLETE_SQL}), TRUE) AS complete
FROM "LiteLLM_DailyUserSpend"
WHERE date >= $1::text AND date <= $2::text
AND ($3::text IS NULL OR api_key = $3::text)
AND ($4::text IS NULL OR user_id = $4::text)
"""
class AutoRouterDailyCosts(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True)
requests: int = 0
llm_spend: float = 0.0
classifier_cost: float = 0.0
classifier_requests: int = 0
estimated_requests: int = 0
estimated_actual_spend: float = 0.0
saved_spend: float = 0.0
complete: bool = True
comparison_complete: bool = True
def matches_savings(self, saved_spend: float) -> bool:
return self.complete and isclose(self.saved_spend, saved_spend, rel_tol=1e-9, abs_tol=1e-9)
@property
def classifier_complete(self) -> bool:
return self.classifier_requests == self.requests
@property
def coverage(self) -> Literal["complete", "partial", "unavailable"]:
if self.complete and self.classifier_complete:
return "complete"
return "partial" if self.requests > 0 else "unavailable"
@property
def recorded_llm_spend(self) -> float | None:
return self.llm_spend if self.complete or self.requests > 0 else None
@property
def recorded_classifier_cost(self) -> float | None:
return self.classifier_cost if self.classifier_complete and self.recorded_llm_spend is not None else None
@property
def recorded_spend(self) -> float | None:
return self.llm_spend + self.classifier_cost if self.recorded_llm_spend is not None else None
def baseline_spend(self, saved_spend: float) -> float | None:
if not self.matches_savings(saved_spend) or not self.classifier_complete or not self.comparison_complete:
return None
if self.requests > 0 and self.estimated_requests == 0:
return None
return self.estimated_actual_spend + saved_spend

View file

@ -0,0 +1,181 @@
import json
import sys
from contextlib import AbstractAsyncContextManager
from datetime import timedelta
from typing import TYPE_CHECKING, Final, Protocol, cast
from litellm.constants import MAX_SPENDLOG_ROWS_TO_QUERY
from litellm.proxy.db.autorouter_daily_spend import DAILY_COSTS_COMPLETE_SQL, AutoRouterDailyCosts
from litellm.proxy.db.create_views import SupportsRawQueries
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
class _ReadTransactions(Protocol):
def tx(self, *, timeout: timedelta, max_wait: timedelta) -> AbstractAsyncContextManager[SupportsRawQueries]: ...
async def recover_daily_router_costs(
prisma_client: "PrismaClient", start_date: str, end_date: str, api_key: str | None, user_id: str | None
) -> AutoRouterDailyCosts | None:
from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
reader: Final = cast(_ReadTransactions, prisma_client.read_db) # cast-ok: untyped Prisma tx delegate
async with reader.tx(timeout=timedelta(seconds=3), max_wait=timedelta(seconds=1)) as transaction:
await transaction.execute_raw("SET TRANSACTION READ ONLY")
await transaction.execute_raw("SET LOCAL statement_timeout = 2000")
rows: Final = await transaction.query_raw(
AUTOROUTER_HISTORICAL_COSTS_SQL, start_date, end_date, api_key, user_id, json.dumps(ROUTE_ENDPOINT_MAPPING)
)
return AutoRouterDailyCosts.model_validate(rows[0]) if rows else None
AUTOROUTER_HISTORICAL_COSTS_SQL: Final = f"""
WITH daily AS MATERIALIZED (
SELECT
date, COALESCE(user_id, '') AS user_id, api_key,
COALESCE(model, '') AS model,
COALESCE(custom_llm_provider, '') AS custom_llm_provider,
COALESCE(mcp_namespaced_tool_name, '') AS mcp_namespaced_tool_name,
COALESCE(endpoint, '') AS endpoint,
SUM(api_requests)::bigint AS api_requests,
SUM(successful_requests)::bigint AS successful_requests,
SUM(failed_requests)::bigint AS failed_requests,
SUM(prompt_tokens)::bigint AS prompt_tokens,
SUM(completion_tokens)::bigint AS completion_tokens,
SUM(spend)::float8 AS spend,
SUM(autorouter_requests)::bigint AS requests,
SUM(autorouter_llm_spend)::float8 AS llm_spend,
SUM(autorouter_classifier_cost)::float8 AS classifier_cost,
SUM(autorouter_classifier_cost_recorded_requests)::bigint AS classifier_requests,
SUM(autorouter_estimated_requests)::bigint AS estimated_requests,
SUM(autorouter_estimated_actual_spend)::float8 AS estimated_actual_spend,
SUM(autorouter_savings_spend)::float8 AS saved_spend,
BOOL_AND({DAILY_COSTS_COMPLETE_SQL}) AS tracked_complete
FROM "LiteLLM_DailyUserSpend"
WHERE date >= $1::text AND date <= $2::text
AND ($3::text IS NULL OR api_key = $3::text)
AND ($4::text IS NULL OR user_id = $4::text)
GROUP BY 1, 2, 3, 4, 5, 6, 7
), limited_logs AS MATERIALIZED (
SELECT
to_char(logs."startTime", 'YYYY-MM-DD') AS date,
COALESCE(logs."user", '') AS user_id, logs.api_key,
COALESCE(logs.model, '') AS model,
COALESCE(logs.custom_llm_provider, '') AS custom_llm_provider,
COALESCE(logs.mcp_namespaced_tool_name, '') AS mcp_namespaced_tool_name,
COALESCE($5::jsonb ->> logs.call_type, '') AS endpoint,
logs.spend, logs.prompt_tokens, logs.completion_tokens, logs.status,
jsonb_build_object(
'internal_call_origin', logs.metadata::jsonb -> 'internal_call_origin',
'status', logs.metadata::jsonb -> 'status',
'routing_decision', logs.metadata::jsonb -> 'routing_decision',
'autorouter_savings', logs.metadata::jsonb -> 'autorouter_savings',
'autorouter_savings_estimate', logs.metadata::jsonb -> 'autorouter_savings_estimate'
) AS metadata
FROM "LiteLLM_SpendLogs" AS logs
WHERE logs."startTime" >= $1::text::timestamp
AND logs."startTime" < $2::text::timestamp + INTERVAL '1 day'
AND ($3::text IS NULL OR logs.api_key = $3::text)
AND ($4::text IS NULL OR logs."user" = $4::text)
AND EXISTS (
SELECT 1 FROM daily
WHERE NOT daily.tracked_complete
AND daily.date = to_char(logs."startTime", 'YYYY-MM-DD')
AND daily.user_id = COALESCE(logs."user", '')
AND daily.api_key = logs.api_key
AND daily.model = COALESCE(logs.model, '')
AND daily.custom_llm_provider = COALESCE(logs.custom_llm_provider, '')
AND daily.mcp_namespaced_tool_name = COALESCE(logs.mcp_namespaced_tool_name, '')
AND daily.endpoint = COALESCE($5::jsonb ->> logs.call_type, '')
)
LIMIT {MAX_SPENDLOG_ROWS_TO_QUERY + 1}
), log_facts AS (
SELECT *,
COALESCE(metadata ->> 'internal_call_origin', '') = '' AS external,
COALESCE(metadata ->> 'status', '') <> 'failure' AS successful,
COALESCE(metadata ->> 'internal_call_origin', '') = ''
AND status = 'success'
AND jsonb_typeof(metadata -> 'routing_decision') = 'object'
AND metadata -> 'routing_decision' <> '{{}}'::jsonb AS routed,
CASE WHEN jsonb_typeof(metadata #> '{{routing_decision,classifier_cost}}') = 'number'
THEN CASE WHEN (metadata #>> '{{routing_decision,classifier_cost}}')::numeric
BETWEEN -{sys.float_info.max} AND {sys.float_info.max}
THEN (metadata #>> '{{routing_decision,classifier_cost}}')::float8 END END AS classifier,
CASE WHEN jsonb_typeof(metadata -> 'autorouter_savings') = 'number'
THEN CASE WHEN (metadata ->> 'autorouter_savings')::numeric
BETWEEN -{sys.float_info.max} AND {sys.float_info.max}
THEN (metadata ->> 'autorouter_savings')::float8 END END AS recorded_savings,
metadata -> 'autorouter_savings_estimate' AS estimate
FROM limited_logs
), classified_logs AS (
SELECT *,
recorded_savings IS NOT NULL AND (
estimate IS NULL OR estimate = 'null'::jsonb OR (
jsonb_typeof(estimate -> 'version') = 'number'
AND estimate ->> 'version' IN ('1', '2', '3')
AND estimate ->> 'status' = 'estimated'
)
) AS estimated,
jsonb_typeof(estimate -> 'version') = 'number'
AND estimate ->> 'version' IN ('1', '2', '3')
AND estimate ->> 'status' = 'unknown' AS unknown
FROM log_facts
), log_totals AS (
SELECT date, user_id, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint,
COUNT(*) FILTER (WHERE external)::bigint AS api_requests,
COUNT(*) FILTER (WHERE external AND successful)::bigint AS successful_requests,
COUNT(*) FILTER (WHERE external AND NOT successful)::bigint AS failed_requests,
SUM(prompt_tokens)::bigint AS prompt_tokens,
SUM(completion_tokens)::bigint AS completion_tokens,
SUM(spend)::float8 AS spend,
COUNT(*) FILTER (WHERE routed)::bigint AS requests,
COALESCE(SUM(spend) FILTER (WHERE routed), 0)::float8 AS llm_spend,
COALESCE(SUM(classifier) FILTER (WHERE routed), 0)::float8 AS classifier_cost,
COUNT(*) FILTER (WHERE routed AND classifier IS NOT NULL)::bigint AS classifier_requests,
COUNT(*) FILTER (WHERE routed AND estimated)::bigint AS estimated_requests,
COALESCE(SUM(spend + COALESCE(classifier, 0)) FILTER (WHERE routed AND estimated), 0)::float8
AS estimated_actual_spend,
COALESCE(SUM(recorded_savings) FILTER (WHERE routed AND estimated), 0)::float8 AS saved_spend,
COALESCE(BOOL_AND(COALESCE(estimated OR unknown, FALSE)) FILTER (WHERE routed), TRUE)
AS comparison_complete
FROM classified_logs
GROUP BY 1, 2, 3, 4, 5, 6, 7
), reconciled AS (
SELECT daily.*, logs.requests AS recovered_requests, logs.llm_spend AS recovered_llm_spend,
logs.classifier_cost AS recovered_classifier_cost,
logs.classifier_requests AS recovered_classifier_requests,
logs.estimated_requests AS recovered_estimated_requests,
logs.estimated_actual_spend AS recovered_estimated_actual_spend,
COALESCE(logs.comparison_complete
AND ABS(logs.saved_spend - daily.saved_spend)
<= GREATEST(1e-9, ABS(daily.saved_spend) * 1e-9), FALSE) AS recovered_comparison_complete,
COALESCE((SELECT COUNT(*) FROM limited_logs) <= {MAX_SPENDLOG_ROWS_TO_QUERY}
AND logs.api_requests = daily.api_requests
AND logs.successful_requests = daily.successful_requests
AND logs.failed_requests = daily.failed_requests
AND logs.prompt_tokens = daily.prompt_tokens
AND logs.completion_tokens = daily.completion_tokens
AND ABS(logs.spend - daily.spend) <= GREATEST(1e-9, ABS(daily.spend) * 1e-9)
AND (daily.saved_spend = 0 OR logs.requests > 0), FALSE) AS recovered
FROM daily LEFT JOIN log_totals AS logs
USING (date, user_id, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint)
)
SELECT
COALESCE(SUM(CASE WHEN recovered THEN recovered_requests ELSE requests END), 0)::bigint AS requests,
COALESCE(SUM(CASE WHEN recovered THEN recovered_llm_spend ELSE llm_spend END), 0)::float8 AS llm_spend,
COALESCE(SUM(CASE WHEN recovered THEN recovered_classifier_cost ELSE classifier_cost END), 0)::float8
AS classifier_cost,
COALESCE(SUM(CASE WHEN recovered THEN recovered_classifier_requests ELSE classifier_requests END), 0)::bigint
AS classifier_requests,
COALESCE(SUM(CASE WHEN recovered THEN recovered_estimated_requests ELSE estimated_requests END), 0)::bigint
AS estimated_requests,
COALESCE(SUM(CASE WHEN recovered THEN recovered_estimated_actual_spend ELSE estimated_actual_spend END), 0)::float8
AS estimated_actual_spend,
COALESCE(SUM(saved_spend), 0)::float8 AS saved_spend,
COALESCE(BOOL_AND(tracked_complete OR recovered), TRUE) AS complete,
COALESCE(BOOL_AND(tracked_complete OR (recovered AND recovered_comparison_complete)), TRUE)
AS comparison_complete
FROM reconciled
"""

View file

@ -99,15 +99,34 @@ _SPEND_COLUMNS: Final = (
"gateway_injected_caching_savings_spend",
"autorouter_savings_spend",
)
_AUTOROUTER_COUNTER_COLUMNS: Final = (
"autorouter_accounted_requests",
"autorouter_requests",
"autorouter_classifier_cost_recorded_requests",
"autorouter_estimated_requests",
)
_AUTOROUTER_SPEND_COLUMNS: Final = (
"autorouter_llm_spend",
"autorouter_classifier_cost",
"autorouter_estimated_actual_spend",
)
_CASTS: Final[Mapping[str, str]] = MappingProxyType(
{
**{column: "bigint" for column in _COUNTER_COLUMNS},
**{column: "double precision" for column in _SPEND_COLUMNS},
**{column: "bigint" for column in (*_COUNTER_COLUMNS, *_AUTOROUTER_COUNTER_COLUMNS)},
**{column: "double precision" for column in (*_SPEND_COLUMNS, *_AUTOROUTER_SPEND_COLUMNS)},
}
)
def _counter_columns(table: DailySpendTable) -> tuple[str, ...]:
return (*_COUNTER_COLUMNS, *(_AUTOROUTER_COUNTER_COLUMNS if table.name == "LiteLLM_DailyUserSpend" else ()))
def _spend_columns(table: DailySpendTable) -> tuple[str, ...]:
return (*_SPEND_COLUMNS, *(_AUTOROUTER_SPEND_COLUMNS if table.name == "LiteLLM_DailyUserSpend" else ()))
def _quoted(columns: Sequence[str]) -> str:
return ", ".join(f'"{column}"' for column in columns)
@ -129,13 +148,13 @@ def conflict_key(table: DailySpendTable, transaction: SpendRow) -> tuple[str, ..
return tuple(_as_text(transaction.get(column)) for column in (table.entity_id_column, *_KEY_COLUMNS))
def _merge(group: Sequence[SpendRow]) -> SpendRow:
def _merge(table: DailySpendTable, group: Sequence[SpendRow]) -> SpendRow:
if len(group) == 1:
return group[0]
return {
**group[0],
**{column: sum(_as_int(row.get(column)) for row in group) for column in _COUNTER_COLUMNS},
**{column: sum(_as_float(row.get(column)) for row in group) for column in _SPEND_COLUMNS},
**{column: sum(_as_int(row.get(column)) for row in group) for column in _counter_columns(table)},
**{column: sum(_as_float(row.get(column)) for row in group) for column in _spend_columns(table)},
}
@ -152,7 +171,9 @@ def merge_by_conflict_key(
key keeps concurrent writers taking row locks in the same sequence.
"""
ordered: Final = sorted(transactions, key=lambda transaction: conflict_key(table, transaction))
return tuple((key, _merge(tuple(group))) for key, group in groupby(ordered, key=lambda t: conflict_key(table, t)))
return tuple(
(key, _merge(table, tuple(group))) for key, group in groupby(ordered, key=lambda t: conflict_key(table, t))
)
def _row_params(
@ -165,8 +186,8 @@ def _row_params(
str(uuid.uuid4()),
*key,
None if transaction.get("model_group") is None else _as_text(transaction.get("model_group")),
*(_as_int(transaction.get(column)) for column in _COUNTER_COLUMNS),
*(_as_float(transaction.get(column)) for column in _SPEND_COLUMNS),
*(_as_int(transaction.get(column)) for column in _counter_columns(table)),
*(_as_float(transaction.get(column)) for column in _spend_columns(table)),
*((None if request_id is None else _as_text(request_id),) if table.carries_request_id else ()),
)
@ -177,8 +198,8 @@ def _insert_columns(table: DailySpendTable) -> tuple[str, ...]:
table.entity_id_column,
*_KEY_COLUMNS,
"model_group",
*_COUNTER_COLUMNS,
*_SPEND_COLUMNS,
*_counter_columns(table),
*_spend_columns(table),
*(("request_id",) if table.carries_request_id else ()),
)
@ -201,7 +222,7 @@ def build_bulk_upsert(
)
increments: Final = ", ".join(
f'"{column}" = {quoted_table}."{column}" + EXCLUDED."{column}"'
for column in (*_COUNTER_COLUMNS, *_SPEND_COLUMNS)
for column in (*_counter_columns(table), *_spend_columns(table))
)
# request_id names one arbitrary contributing request, so an entry carrying none must
# not blank out the one already recorded.

View file

@ -74,9 +74,11 @@ from litellm.proxy.spend_tracking.compression_savings import (
extract_compression_saved_tokens,
)
from litellm.proxy.spend_tracking.savings import (
classifier_cost_from_decision,
compute_savings_spend,
extract_cache_creation_tokens,
extract_cache_read_tokens,
known_autorouter_savings,
marks_gateway_injection,
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
@ -2621,6 +2623,29 @@ class DBSpendUpdateWriter:
endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None)
is_internal_call: Final = bool(_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY))
routing_decision: Final = _metadata.get("routing_decision")
is_routed_request: Final = (
not is_internal_call
and payload.get("status") == "success"
and isinstance(routing_decision, Mapping)
and bool(routing_decision)
)
classifier_cost: Final = classifier_cost_from_decision(routing_decision) if is_routed_request else None
known_savings: Final = (
known_autorouter_savings(
model=payload.get("model"),
custom_llm_provider=payload.get("custom_llm_provider"),
routing_decision=routing_decision,
usage_object=usage_obj,
model_id=payload.get("model_id"),
llm_router=get_llm_router,
cost_breakdown=_metadata.get("cost_breakdown"),
recorded_autorouter_savings=_metadata.get("autorouter_savings"),
recorded_autorouter_savings_estimate=_metadata.get("autorouter_savings_estimate"),
)
if is_routed_request
else None
)
cache_read_input_tokens: Final = extract_cache_read_tokens(usage_obj)
compression_saved_tokens: Final = extract_compression_saved_tokens(_metadata)
savings_spend: Final = compute_savings_spend(
@ -2665,6 +2690,15 @@ class DBSpendUpdateWriter:
prompt_caching_savings_spend=savings_spend.prompt_caching,
gateway_injected_caching_savings_spend=savings_spend.gateway_injected_caching,
autorouter_savings_spend=0.0 if is_internal_call else savings_spend.autorouter,
autorouter_accounted_requests=0 if is_internal_call else 1,
autorouter_requests=int(is_routed_request),
autorouter_llm_spend=payload["spend"] if is_routed_request else 0.0,
autorouter_classifier_cost=classifier_cost or 0.0,
autorouter_classifier_cost_recorded_requests=int(classifier_cost is not None),
autorouter_estimated_requests=int(known_savings is not None),
autorouter_estimated_actual_spend=(
payload["spend"] + (classifier_cost or 0.0) if known_savings is not None else 0.0
),
total_response_time_ms=timed_duration_ms or 0,
timed_requests=0 if timed_duration_ms is None else 1,
)

View file

@ -149,6 +149,27 @@ class DailySpendUpdateQueue(BaseUpdateQueue):
daily_transaction["timed_requests"] = (
payload.get("timed_requests", 0) or 0
) + daily_transaction.get("timed_requests", 0)
aggregated_daily_spend_update_transactions[_key] = {
**daily_transaction,
"autorouter_accounted_requests": daily_transaction.get("autorouter_accounted_requests", 0)
+ payload.get("autorouter_accounted_requests", 0),
"autorouter_requests": daily_transaction.get("autorouter_requests", 0)
+ payload.get("autorouter_requests", 0),
"autorouter_llm_spend": daily_transaction.get("autorouter_llm_spend", 0.0)
+ payload.get("autorouter_llm_spend", 0.0),
"autorouter_classifier_cost": daily_transaction.get("autorouter_classifier_cost", 0.0)
+ payload.get("autorouter_classifier_cost", 0.0),
"autorouter_classifier_cost_recorded_requests": daily_transaction.get(
"autorouter_classifier_cost_recorded_requests", 0
)
+ payload.get("autorouter_classifier_cost_recorded_requests", 0),
"autorouter_estimated_requests": daily_transaction.get("autorouter_estimated_requests", 0)
+ payload.get("autorouter_estimated_requests", 0),
"autorouter_estimated_actual_spend": daily_transaction.get(
"autorouter_estimated_actual_spend", 0.0
)
+ payload.get("autorouter_estimated_actual_spend", 0.0),
}
else:
aggregated_daily_spend_update_transactions[_key] = deepcopy(payload)

View file

@ -949,6 +949,7 @@ class PrismaManager:
prisma_command_timeout,
run_prisma,
)
from litellm_proxy_extras.utils import ProxyExtrasDBManager
except ImportError as e:
verbose_proxy_logger.error("\x1b[1;31mLiteLLM: Failed to import proxy extras. Got %s\x1b[0m", e)
return False
@ -967,6 +968,7 @@ class PrismaManager:
stdout=None,
stderr=None,
)
ProxyExtrasDBManager.apply_autorouter_daily_coverage()
PrismaManager._apply_replica_identity_full_if_requested()
return True
except subprocess.TimeoutExpired as e:

View file

@ -31,6 +31,8 @@ from litellm.proxy.auth.auth_checks import (
can_key_call_resolved_model,
)
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.db.autorouter_daily_spend import AUTOROUTER_DAILY_COSTS_SQL, AutoRouterDailyCosts
from litellm.proxy.db.autorouter_historical_spend import recover_daily_router_costs
from litellm.proxy.db.autorouter_session_rollup import (
AUTOROUTER_BENCHMARKS_SQL,
bounded_session_id,
@ -686,10 +688,31 @@ def _savings_cohort(
return saved_spend, actual_spend + saved_spend
def _recorded_baseline_spend(estimated_turns: int, actual_spend: float, saved_spend: float) -> float | None:
if estimated_turns == 0 and saved_spend == 0:
return None
return actual_spend + saved_spend
async def _daily_router_costs(
prisma_client: "PrismaClient", start_date: str, end_date: str, api_key: str | None, user_id: str | None
) -> AutoRouterDailyCosts:
rows: Final = await _query_raw(prisma_client, AUTOROUTER_DAILY_COSTS_SQL, start_date, end_date, api_key, user_id)
recorded: Final = AutoRouterDailyCosts.model_validate(rows[0]) if rows else AutoRouterDailyCosts()
if recorded.complete:
return recorded
try:
recovered: Final = await recover_daily_router_costs(prisma_client, start_date, end_date, api_key, user_id)
return recovered if recovered is not None else recorded
except Exception: # noqa: BLE001 # optional recovery must preserve durable daily savings and costs
verbose_proxy_logger.warning("Auto-router historical cost recovery unavailable; using recorded daily costs")
return recorded
def _benchmark_totals(row: _SessionAggRow) -> AutoRouterBenchmarkTotals:
return_misses: Final = row.return_turns - row.return_hits
saved_spend, baseline_spend = _savings_cohort(
row.turns, row.savings_estimated_turns, row.savings_estimated_actual_spend, row.savings_estimated_saved_spend
baseline_spend: Final = (
_recorded_baseline_spend(row.savings_estimated_turns, row.spend, row.saved_spend) if row.turns else 0.0
)
sessions: Final = row.sessions
return AutoRouterBenchmarkTotals(
@ -699,15 +722,16 @@ def _benchmark_totals(row: _SessionAggRow) -> AutoRouterBenchmarkTotals:
avg_session_seconds=row.session_seconds / sessions if sessions else 0.0,
avg_tokens_per_session=row.total_tokens / sessions if sessions else 0.0,
spend=row.spend,
llm_spend=row.spend - row.classifier_cost if row.classifier_cost_recorded_turns == row.turns else None,
cost_coverage="complete" if row.classifier_cost_recorded_turns == row.turns else "partial",
cost_requests=row.turns,
savings_estimated_turns=row.savings_estimated_turns,
savings_estimated_actual_spend=row.savings_estimated_actual_spend,
saved_spend=saved_spend,
saved_spend=row.saved_spend,
classifier_cost=row.classifier_cost if row.classifier_cost_recorded_turns == row.turns else None,
baseline_spend=baseline_spend,
saved_pct=_pct(saved_spend, baseline_spend) if saved_spend is not None and baseline_spend is not None else None,
saved_per_session=(row.savings_estimated_saved_spend / sessions if sessions else 0.0)
if row.savings_estimated_turns == row.turns
else None,
saved_pct=_pct(row.saved_spend, baseline_spend) if baseline_spend is not None else None,
saved_per_session=row.saved_spend / sessions if sessions else 0.0,
cache=AutoRouterCacheStats(
coverage_pct=_pct(row.covered_turns, row.turns),
hit_rate_pct=_pct(row.cache_hits, row.covered_turns),
@ -736,6 +760,9 @@ def _benchmark_group(row: _SessionAggRow) -> AutoRouterBenchmarkGroup:
avg_session_seconds=totals.avg_session_seconds,
avg_tokens_per_session=totals.avg_tokens_per_session,
spend=totals.spend,
llm_spend=totals.llm_spend,
cost_coverage=totals.cost_coverage,
cost_requests=totals.cost_requests,
saved_spend=totals.saved_spend,
savings_estimated_turns=totals.savings_estimated_turns,
savings_estimated_actual_spend=totals.savings_estimated_actual_spend,
@ -847,12 +874,12 @@ async def get_auto_router_benchmarks(
Benchmarks for the auto-router dashboard: session shape, savings against the configured
baseline, and prompt-caching behaviour bucketed by what the router did.
Reads session rollups folded once per request at spend-write time, so this endpoint
never scans LiteLLM_SpendLogs. A user filter selects only turns attributed to that
internal user when written; older key-only history remains outside user views. A session
is in the window when it overlaps it: its last turn is on or after start_date and its first turn is on or before
end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is
over that bucket's turns.
Total savings use the same request-date daily aggregation as Overall cost optimization,
including recorded history and requests without session IDs. Costs use matching request
dates with explicit coverage for history predating daily router cost tracking. Session
statistics and per-router groups cover whole sessions that overlap the window.
A user filter uses the user recorded when each row was written.
Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is over its turns.
The rollup supplies the measures, never the list. Which routers appear comes from the
model registry, so one shows up as soon as it is configured and reads zero until it
@ -874,6 +901,20 @@ async def get_auto_router_benchmarks(
if end_day < start_day:
raise HTTPException(status_code=400, detail="end_date must not be earlier than start_date")
from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity_aggregated
daily: Final = await get_daily_activity_aggregated(
prisma_client=prisma_client,
table_name="litellm_dailyuserspend",
entity_id_field="user_id",
entity_id=user_id,
entity_metadata_field=None,
start_date=start_day.strftime("%Y-%m-%d"),
end_date=end_day.strftime("%Y-%m-%d"),
model=None,
api_key=api_key,
)
raw_rows: Final = await _query_raw(
prisma_client,
AUTOROUTER_BENCHMARKS_SQL,
@ -887,11 +928,36 @@ async def get_auto_router_benchmarks(
*(_benchmark_group(row) for row in rows),
*_idle_router_groups(llm_router, frozenset((row.router_name, row.router_type) for row in rows)),
)
session_totals: Final = _benchmark_totals(_summed_agg_row(rows))
costs: Final = await _daily_router_costs(
prisma_client,
start_day.strftime("%Y-%m-%d"),
end_day.strftime("%Y-%m-%d"),
api_key,
user_id,
)
saved_spend: Final = daily.metadata.total_autorouter_savings_spend
baseline_spend: Final = costs.baseline_spend(saved_spend)
return AutoRouterBenchmarksResponse(
start_date=start_day.strftime("%Y-%m-%d"),
end_date=end_day.strftime("%Y-%m-%d"),
routers_in_scope=len(groups),
totals=_benchmark_totals(_summed_agg_row(rows)),
totals=session_totals.model_copy(
update=MappingProxyType(
{
"saved_spend": saved_spend,
"spend": costs.recorded_spend,
"llm_spend": costs.recorded_llm_spend,
"classifier_cost": costs.recorded_classifier_cost,
"cost_coverage": costs.coverage,
"cost_requests": costs.requests if costs.complete else None,
"savings_estimated_turns": costs.estimated_requests,
"savings_estimated_actual_spend": costs.estimated_actual_spend,
"baseline_spend": baseline_spend,
"saved_pct": _pct(saved_spend, baseline_spend) if baseline_spend is not None else None,
}
)
),
groups=groups,
)
@ -927,7 +993,7 @@ async def get_auto_router_session(
raise HTTPException(
status_code=404, detail=f"No auto-routed turns recorded for session {session_id!r} under this key"
)
saved_spend, baseline_spend = _savings_cohort(
_, estimated_baseline_spend = _savings_cohort(
row.turns, row.savings_estimated_turns, row.savings_estimated_actual_spend, row.savings_estimated_saved_spend
)
return AutoRouterSessionResponse(
@ -939,11 +1005,11 @@ async def get_auto_router_session(
spend=row.spend,
savings_estimated_turns=row.savings_estimated_turns,
savings_estimated_actual_spend=row.savings_estimated_actual_spend,
saved_spend=saved_spend,
baseline_spend=baseline_spend if row.savings_estimated_turns == row.turns else None,
savings_estimated_baseline_spend=baseline_spend,
saved_spend=row.saved_spend,
baseline_spend=_recorded_baseline_spend(row.savings_estimated_turns, row.spend, row.saved_spend),
savings_estimated_baseline_spend=estimated_baseline_spend,
baseline_model=row.baseline_model,
baseline_models=row.savings_estimated_baseline_models,
baseline_models=row.baseline_models,
)

View file

@ -805,6 +805,13 @@ model LiteLLM_DailyUserSpend {
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
autorouter_accounted_requests BigInt @default(0)
autorouter_requests BigInt @default(0)
autorouter_llm_spend Float @default(0.0)
autorouter_classifier_cost Float @default(0.0)
autorouter_classifier_cost_recorded_requests BigInt @default(0)
autorouter_estimated_requests BigInt @default(0)
autorouter_estimated_actual_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -1579,6 +1586,7 @@ model LiteLLM_AutoRouterBaselineObservation {
revision BigInt
data String
publication String?
daily_costs_publication String?
conflicted Boolean @default(false)
@@index([scope, started_at, request_id], map: "idx_autorouter_baseline_event_order")

View file

@ -628,6 +628,42 @@ def prompt_caching_savings_for_request(
return _prompt_caching_savings(request_pricing[1], request_pricing[0], usage_object, cost_breakdown, billed_at)
def known_autorouter_savings(
model: str | None,
custom_llm_provider: str | None,
routing_decision: Mapping[str, object] | None,
usage_object: Mapping[str, object] | None,
model_id: str | None = None,
llm_router: "Callable[[], Router | None] | None" = None,
cost_breakdown: Mapping[str, object] | None = None,
recorded_autorouter_savings: object = None,
recorded_autorouter_savings_estimate: Mapping[str, object] | None = None,
) -> float | None:
recorded: Final = (
recorded_estimated_autorouter_savings(
MappingProxyType(
{
"autorouter_savings": recorded_autorouter_savings,
"autorouter_savings_estimate": recorded_autorouter_savings_estimate,
}
)
)
if recorded_autorouter_savings_estimate is not None
else _numeric_savings(recorded_autorouter_savings)
)
if recorded is not None or recorded_autorouter_savings_estimate is not None:
return recorded
return autorouter_savings_for_request(
model=model,
custom_llm_provider=custom_llm_provider,
routing_decision=routing_decision,
usage_object=usage_object,
model_id=model_id,
llm_router=llm_router,
cost_breakdown=cost_breakdown,
)
def compute_savings_spend(
model: str | None,
custom_llm_provider: str | None,
@ -697,32 +733,16 @@ def compute_savings_spend(
prompt_caching: Final = _prompt_caching_savings(pricing, provider, usage_object, cost_breakdown, billed_at) or 0.0
gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0
# The figure the logging path recorded wins, before the usage gate on purpose: a row
# whose usage no longer parses still carries the number computed when it did.
recorded_savings: Final = (
recorded_estimated_autorouter_savings(
MappingProxyType(
{
"autorouter_savings": recorded_autorouter_savings,
"autorouter_savings_estimate": recorded_autorouter_savings_estimate,
}
)
)
if recorded_autorouter_savings_estimate is not None
else _numeric_savings(recorded_autorouter_savings)
)
autorouter: Final = (
recorded_savings
if recorded_savings is not None or recorded_autorouter_savings_estimate is not None
else autorouter_savings_for_request(
model=model,
custom_llm_provider=custom_llm_provider,
routing_decision=routing_decision,
usage_object=usage_object,
model_id=model_id,
llm_router=llm_router,
cost_breakdown=cost_breakdown,
)
autorouter: Final = known_autorouter_savings(
model=model,
custom_llm_provider=custom_llm_provider,
routing_decision=routing_decision,
usage_object=usage_object,
model_id=model_id,
llm_router=llm_router,
cost_breakdown=cost_breakdown,
recorded_autorouter_savings=recorded_autorouter_savings,
recorded_autorouter_savings_estimate=recorded_autorouter_savings_estimate,
)
return SavingsSpend(
compression=compression,

View file

@ -211,37 +211,56 @@ class AutoRouterCacheStats(BaseModel):
class AutoRouterBenchmarkTotals(BaseModel):
"""Session-shape and savings aggregates over auto-routed traffic in the window."""
"""Request-date savings totals with statistics for whole sessions overlapping the window."""
sessions: int
turns: int
avg_turns_per_session: float
avg_session_seconds: float
avg_tokens_per_session: float
spend: float = Field(description="What the routed traffic actually cost")
spend: float | None = Field(
description="Actual auto-router cost including classification. Totals use request dates; groups cover whole "
"overlapping sessions. Partial recorded history is identified by cost_coverage"
)
classifier_cost: float | None = Field(
description="Recorded LLM classifier cost already included in spend; null when any session turns predate "
"subtotal recording, and zero for an empty window"
description="Classifier charges included in actual cost; null when any contributing request lacks the subtotal"
)
llm_spend: float | None = Field(default=None, description="Routed LLM cost excluding classifier charges")
cost_coverage: Literal["complete", "partial", "unavailable"] = Field(
default="unavailable", description="Whether recorded costs cover the entire selected population"
)
cost_requests: int | None = Field(
default=None,
description="Number of requests in the cost population; null when historical coverage is incomplete",
)
savings_estimated_turns: int = Field(
description="Turns covered by the current savings estimator; legacy estimates are excluded"
description="Requests with an available savings comparison. Totals use request dates and groups use session "
"coverage. Historical savings remain included even when their coverage was not tracked"
)
savings_estimated_actual_spend: float = Field(
description="Actual spend, including classifier cost, for covered turns only"
description="Actual spend including classification for requests covered by the savings comparison"
)
saved_spend: float | None = Field(
description="Signed savings for covered turns only; null when traffic has no current estimates"
description="Signed recorded savings, including historical and current estimates once. Totals use the same "
"request-date daily source as Overall cost optimization, including requests without sessions. Per-router "
"groups cover whole overlapping sessions. Turns without an estimate add no savings"
)
baseline_spend: float | None = Field(
description="Matching actual spend plus recorded savings. Daily totals compare estimated requests only; "
"null when historical cost coverage or the savings estimate is unavailable"
)
saved_pct: float | None = Field(
description="Recorded savings divided by the matching estimated baseline; null when that baseline is unavailable"
)
baseline_spend: float | None = Field(description="Estimated single-model cost for covered turns only")
saved_pct: float | None = Field(description="Covered savings over covered baseline spend, as a percentage")
saved_per_session: float | None = Field(
description="Average session savings; unavailable unless every turn is covered"
description="Recorded savings in whole overlapping sessions divided by their session count, independent "
"of request-date daily savings totals"
)
cache: AutoRouterCacheStats
class AutoRouterBenchmarkGroup(AutoRouterBenchmarkTotals):
"""One auto-router's slice of the benchmarks."""
"""One auto-router's whole overlapping sessions, separate from request-date daily totals."""
router_name: str = Field(description="The auto-router alias requests were sent to")
router_type: str = Field(description="complexity, adaptive or quality")
@ -273,27 +292,29 @@ class AutoRouterSessionResponse(BaseModel):
savings_estimated_actual_spend: float = Field(
description="Actual spend, including classifier cost, for covered turns only"
)
saved_spend: float | None = Field(description="Estimated savings for covered turns only, net of classifier cost")
saved_spend: float | None = Field(
description="Recorded signed savings, retaining historical estimates and adding current estimates once"
)
baseline_spend: float | None = Field(
description="Estimated single-model cost; unavailable unless every turn is covered"
description="Actual session spend plus recorded savings; turns without an estimate add no savings"
)
savings_estimated_baseline_spend: float | None = Field(
description="Estimated single-model cost for covered turns only"
)
baseline_model: str | None = Field(
description="The savings baseline most covered turns were priced against, recorded turn by "
description="The savings baseline recorded by most session turns, including historical turns, recorded turn by "
"turn, so it still names the counterfactual after the router is reconfigured or removed. None when no "
"turn recorded one: rows from before the baseline was recorded, and adaptive and quality routers, "
"which derive no baseline and so report no savings"
)
baseline_models: Mapping[str, int] = Field(
description="Covered turns priced against each baseline model; more than one entry means the router's "
"baseline changed mid-session and baseline_spend mixes both"
description="Session turns recording each baseline model, including historical turns; more than one entry "
"means the router's baseline changed mid-session. These counts do not imply savings-estimate coverage"
)
class AutoRouterBenchmarksResponse(BaseModel):
"""Benchmarks for the auto-router dashboard, aggregated from the per-session rollup."""
"""Daily recorded savings with tracked-session breakdowns for the auto-router dashboard."""
start_date: str = Field(description="Window start day, YYYY-MM-DD UTC, inclusive")
end_date: str = Field(description="Window end day, YYYY-MM-DD UTC, inclusive")

View file

@ -805,6 +805,13 @@ model LiteLLM_DailyUserSpend {
prompt_caching_savings_spend Float @default(0.0)
gateway_injected_caching_savings_spend Float @default(0.0)
autorouter_savings_spend Float @default(0.0)
autorouter_accounted_requests BigInt @default(0)
autorouter_requests BigInt @default(0)
autorouter_llm_spend Float @default(0.0)
autorouter_classifier_cost Float @default(0.0)
autorouter_classifier_cost_recorded_requests BigInt @default(0)
autorouter_estimated_requests BigInt @default(0)
autorouter_estimated_actual_spend Float @default(0.0)
spend Float @default(0.0)
api_requests BigInt @default(0)
successful_requests BigInt @default(0)
@ -1579,6 +1586,7 @@ model LiteLLM_AutoRouterBaselineObservation {
revision BigInt
data String
publication String?
daily_costs_publication String?
conflicted Boolean @default(false)
@@index([scope, started_at, request_id], map: "idx_autorouter_baseline_event_order")

View file

@ -3,6 +3,7 @@ import os
import re
import sys
from pathlib import Path
from typing import Final
import pytest
@ -740,16 +741,28 @@ ERROR: relation "SomeTable" already exists
),
),
)
def test_v2_migrations_use_the_direct_connection_with_the_runtime_schema(pooled, direct, expected):
def test_v2_migrations_use_the_direct_connection_with_the_runtime_schema(
monkeypatch: pytest.MonkeyPatch, pooled: str, direct: str | None, expected: str
) -> None:
from unittest.mock import Mock
import litellm_proxy_extras.utils as utils_module
from litellm_proxy_extras.migration_lock import migration_environment
environment = {"DATABASE_URL": pooled, "PRISMA_OFFLINE_MODE": "true"}
configured = {**environment, **({"DIRECT_URL": direct} if direct else {})}
migrated = migration_environment(configured)
environment: Final = {"DATABASE_URL": pooled, "PRISMA_OFFLINE_MODE": "true"}
configured: Final = {**environment, **({"DIRECT_URL": direct} if direct else {})}
migrated: Final = migration_environment(configured)
run_prisma: Final = Mock()
monkeypatch.setattr(utils_module, "_get_prisma_env", lambda: configured)
monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", run_prisma)
ProxyExtrasDBManager.apply_autorouter_daily_coverage()
assert migrated["DATABASE_URL"] == expected
assert migrated["PRISMA_OFFLINE_MODE"] == "true"
assert configured["DATABASE_URL"] == pooled
run_prisma.assert_called_once()
assert run_prisma.call_args.kwargs["env"] == migrated
class _MigrateDeployHarness:

View file

@ -1,12 +1,20 @@
"""Session-scoped Prisma client for spend-rollup behavior tests against a real Postgres."""
import asyncio
from collections.abc import AsyncIterator
from typing import Final
import pytest_asyncio
from litellm_proxy_extras.utils import ProxyExtrasDBManager
from prisma import Prisma
@pytest_asyncio.fixture(scope="session", loop_scope="session")
async def db():
client = Prisma()
async def db() -> AsyncIterator[Prisma]:
await asyncio.to_thread(ProxyExtrasDBManager.apply_autorouter_daily_coverage)
client: Final = Prisma()
await client.connect()
yield client
await client.disconnect()
try:
yield client
finally:
await client.disconnect()

View file

@ -0,0 +1,273 @@
import json
import os
import time
import uuid
from collections.abc import Mapping
from datetime import timedelta
from typing import Final
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
import pytest
from prisma import Prisma
from prisma.errors import RawQueryError
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.db.autorouter_daily_spend import AUTOROUTER_DAILY_COSTS_SQL, AutoRouterDailyCosts
from litellm.proxy.db.autorouter_historical_spend import AUTOROUTER_HISTORICAL_COSTS_SQL, recover_daily_router_costs
from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, build_bulk_upsert, merge_by_conflict_key
from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
from litellm.proxy.utils import PrismaClient, ProxyLogging
pytestmark = pytest.mark.asyncio(loop_scope="session")
def _daily(day: str, key: str, user: str, llm: float, classifier: float, saved: float) -> Mapping[str, object]:
return {
"date": day,
"api_key": key,
"user_id": user,
"model": "model",
"custom_llm_provider": "provider",
"spend": llm,
"api_requests": 1,
"successful_requests": 1,
"autorouter_accounted_requests": 1,
"autorouter_requests": 1,
"autorouter_llm_spend": llm,
"autorouter_classifier_cost": classifier,
"autorouter_classifier_cost_recorded_requests": 1,
"autorouter_estimated_requests": 1,
"autorouter_estimated_actual_spend": llm + classifier,
"autorouter_savings_spend": saved,
}
async def test_daily_costs_use_request_dates_and_filters_instead_of_session_boundaries(db: Prisma) -> None:
async with db.tx() as tx:
await tx.execute_raw(
'CREATE TEMP TABLE "LiteLLM_DailyUserSpend" (LIKE public."LiteLLM_DailyUserSpend" INCLUDING ALL) ON COMMIT DROP'
)
rows: Final = (
_daily("2026-09-21", "a", "owner-a", 8.0, 0.0, 4.0),
_daily("2026-09-22", "a", "owner-a", 1.8, 0.2, 1.0),
_daily("2026-09-22", "a", "owner-a", 3.0, 0.0, 2.0),
_daily("2026-09-22", "b", "owner-b", 7.0, 0.0, 5.0),
)
table: Final = DAILY_SPEND_TABLES["user"]
sql, values = build_bulk_upsert(table, merge_by_conflict_key(table, rows))
await tx.execute_raw(sql, *values)
for key, user, expected_spend, saved, requests in (
(None, None, 12.0, 8.0, 3),
("a", None, 5.0, 3.0, 2),
(None, "owner-a", 5.0, 3.0, 2),
("a", "owner-b", 0.0, 0.0, 0),
):
result: Final = await tx.query_raw(AUTOROUTER_DAILY_COSTS_SQL, "2026-09-22", "2026-09-22", key, user)
costs: Final = AutoRouterDailyCosts.model_validate(result[0])
assert costs.complete and costs.requests == requests
assert costs.recorded_spend == pytest.approx(expected_spend)
assert costs.saved_spend == saved
assert costs.baseline_spend(saved) == pytest.approx(expected_spend + saved)
async def test_an_old_writer_cannot_certify_partial_daily_costs_as_complete(db: Prisma) -> None:
async with db.tx() as tx:
await tx.execute_raw(
'CREATE TEMP TABLE "LiteLLM_DailyUserSpend" (LIKE public."LiteLLM_DailyUserSpend" INCLUDING ALL) ON COMMIT DROP'
)
table: Final = DAILY_SPEND_TABLES["user"]
row: Final = _daily("2026-09-22", "a", "owner", 2.0, 0.1, 3.0)
sql, values = build_bulk_upsert(table, merge_by_conflict_key(table, (row,)))
await tx.execute_raw(sql, *values)
await tx.execute_raw(
'UPDATE "LiteLLM_DailyUserSpend" SET api_requests=api_requests+1, successful_requests=successful_requests+1, '
"spend=spend+8, autorouter_savings_spend=autorouter_savings_spend+4"
)
result: Final = await tx.query_raw(AUTOROUTER_DAILY_COSTS_SQL, "2026-09-22", "2026-09-22", None, None)
costs: Final = AutoRouterDailyCosts.model_validate(result[0])
assert costs.coverage == "partial"
assert costs.recorded_spend == pytest.approx(2.1)
assert costs.saved_spend == 7.0
assert costs.baseline_spend(7.0) is None
async def test_retained_historical_costs_reconcile_through_prisma_and_reject_a_missing_free_request(db: Prisma) -> None:
async with db.tx() as tx:
await tx.execute_raw(
'CREATE TEMP TABLE "LiteLLM_DailyUserSpend" (LIKE public."LiteLLM_DailyUserSpend" INCLUDING ALL) ON COMMIT DROP'
)
await tx.execute_raw(
'CREATE TEMP TABLE "LiteLLM_SpendLogs" (LIKE public."LiteLLM_SpendLogs" INCLUDING ALL) ON COMMIT DROP'
)
cases: Final = (
("2026-09-21", "a", "owner-a", 90.0, 1.0, 9.0),
("2026-09-22", "a", "owner-a", 2.0, 0.2, 3.0),
("2026-09-22", "b", "owner-b", 7.0, 0.5, 5.0),
)
rows: Final = tuple(
{
"date": day,
"api_key": key,
"user_id": user,
"model": "removed-historical-model",
"custom_llm_provider": "removed-provider",
"endpoint": "/responses",
"spend": llm + classifier,
"api_requests": 2,
"successful_requests": 2,
"prompt_tokens": 25,
"completion_tokens": 10,
"autorouter_savings_spend": saved,
}
for day, key, user, llm, classifier, saved in cases
)
table: Final = DAILY_SPEND_TABLES["user"]
sql, values = build_bulk_upsert(table, merge_by_conflict_key(table, rows))
await tx.execute_raw(sql, *values)
logs: Final = tuple(
{
"request_id": f"{day}-{key}-{suffix}",
"started_at": f"{day}T12:00:00",
"api_key": key,
"user_id": user,
"spend": spend,
"prompt_tokens": prompt,
"completion_tokens": completion,
"metadata": metadata,
}
for day, key, user, llm, classifier, saved in cases
for suffix, spend, prompt, completion, metadata in (
(
"routed",
llm,
20,
10,
{
"routing_decision": {"router_model_name": "retired-router", "classifier_cost": classifier},
"autorouter_savings": saved,
},
),
("free", 0.0, 0, 0, {}),
(
"classifier",
classifier,
5,
0,
{
"internal_call_origin": "autorouter_classifier",
"routing_decision": {"router_model_name": "retired-router", "classifier_cost": classifier},
},
),
)
)
await tx.execute_raw(
"""INSERT INTO "LiteLLM_SpendLogs"
(request_id, "startTime", "endTime", call_type, api_key, "user", model,
custom_llm_provider, spend, prompt_tokens, completion_tokens, status, metadata)
SELECT request_id, started_at::timestamp, started_at::timestamp, 'aresponses', api_key, user_id,
'removed-historical-model', 'removed-provider', spend, prompt_tokens, completion_tokens,
'success', metadata
FROM jsonb_to_recordset($1::jsonb) AS rows(
request_id text, started_at text, api_key text, user_id text,
spend float8, prompt_tokens int, completion_tokens int, metadata jsonb
)""",
json.dumps(logs),
)
for key, user, actual, saved, requests in (
(None, None, 9.7, 8.0, 2),
("a", None, 2.2, 3.0, 1),
(None, "owner-a", 2.2, 3.0, 1),
("a", "owner-a", 2.2, 3.0, 1),
("a", "owner-b", 0.0, 0.0, 0),
):
result: Final = await tx.query_raw(
AUTOROUTER_HISTORICAL_COSTS_SQL,
"2026-09-22",
"2026-09-22",
key,
user,
json.dumps(ROUTE_ENDPOINT_MAPPING),
)
recovered: Final = AutoRouterDailyCosts.model_validate(result[0])
assert recovered.complete and recovered.comparison_complete
assert recovered.requests == requests
assert recovered.recorded_spend == pytest.approx(actual)
assert recovered.saved_spend == saved
assert recovered.baseline_spend(saved) == pytest.approx(actual + saved)
await tx.execute_raw('DELETE FROM "LiteLLM_SpendLogs" WHERE request_id = $1', "2026-09-22-a-free")
incomplete: Final = await tx.query_raw(
AUTOROUTER_HISTORICAL_COSTS_SQL,
"2026-09-22",
"2026-09-22",
"a",
None,
json.dumps(ROUTE_ENDPOINT_MAPPING),
)
costs: Final = AutoRouterDailyCosts.model_validate(incomplete[0])
assert not costs.complete
assert costs.saved_spend == 3.0
assert costs.recorded_spend is None
assert costs.baseline_spend(3.0) is None
async def test_blocked_historical_recovery_times_out_without_changing_costs_or_read_connection(
db: Prisma, monkeypatch: pytest.MonkeyPatch
) -> None:
identity: Final = uuid.uuid4().hex
database_url: Final = os.environ["DATABASE_URL"]
parsed: Final = urlsplit(database_url)
query: Final = urlencode({**dict(parse_qsl(parsed.query)), "connection_limit": "1"})
monkeypatch.setenv("DATABASE_URL_READ_REPLICA", urlunsplit(parsed._replace(query=query)))
client: Final = PrismaClient(database_url, ProxyLogging(UserApiKeyCache()))
row: Final = {
**_daily("2026-09-22", identity, identity, 2.0, 0.1, 7.0),
"api_requests": 2,
"successful_requests": 2,
"spend": 10.0,
}
table: Final = DAILY_SPEND_TABLES["user"]
statement, values = build_bulk_upsert(table, merge_by_conflict_key(table, (row,)))
await db.execute_raw(statement, *values)
try:
await client.db.connect()
await client.read_db.execute_raw("SET statement_timeout = 4500")
connection_query: Final = (
"SELECT pg_backend_pid() AS pid, current_setting('statement_timeout') AS timeout, "
"current_setting('transaction_read_only') AS read_only"
)
connection_before: Final = await client.read_db.query_raw(connection_query)
daily_before: Final = await client.read_db.query_raw(
AUTOROUTER_DAILY_COSTS_SQL,
"2026-09-22",
"2026-09-22",
identity,
identity,
)
recorded: Final = AutoRouterDailyCosts.model_validate(daily_before[0])
assert recorded.recorded_spend == pytest.approx(2.1) and recorded.saved_spend == 7.0
assert recorded.coverage == "partial"
async with db.tx(timeout=timedelta(seconds=8)) as blocker:
await blocker.execute_raw('LOCK TABLE "LiteLLM_SpendLogs" IN ACCESS EXCLUSIVE MODE')
started: Final = time.monotonic()
with pytest.raises(RawQueryError, match="canceling statement due to statement timeout"):
await recover_daily_router_costs(client, "2026-09-22", "2026-09-22", identity, identity)
assert time.monotonic() - started < 3.5
assert await client.read_db.query_raw(connection_query) == connection_before
assert (
await client.read_db.query_raw(
AUTOROUTER_DAILY_COSTS_SQL,
"2026-09-22",
"2026-09-22",
identity,
identity,
)
== daily_before
)
recovered: Final = await recover_daily_router_costs(client, "2026-09-22", "2026-09-22", identity, identity)
assert recovered is not None
assert recovered.recorded_spend == recorded.recorded_spend and recovered.saved_spend == recorded.saved_spend
assert await client.read_db.query_raw(connection_query) == connection_before
finally:
await client.db.disconnect()
await db.execute_raw('DELETE FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1', identity)

View file

@ -15,10 +15,12 @@ from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
from litellm.proxy.db.baseline_accounting import (
BaselineAccountingRecord,
BaselineAccountingStore,
BaselinePublication,
DailyBaselineAttribution,
DailyBaselineTarget,
)
from litellm.proxy.db.create_views import SupportsRawQueries
from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, build_bulk_upsert, merge_by_conflict_key
from litellm.proxy.spend_tracking.baseline_accounting import BaselineObservation
from litellm.proxy.spend_tracking.savings import BaselineCostSnapshot
from litellm.types.utils import Usage
@ -107,6 +109,11 @@ async def _user_sessions(db: Prisma, record: BaselineAccountingRecord) -> dict[s
return {str(row["user_id"]): row for row in rows}
async def _daily_user(db: Prisma, record: BaselineAccountingRecord) -> dict[str, object]:
rows: Final = await db.query_raw('SELECT * FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1', record.api_key)
return rows[0]
async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
store: Final = _store(db)
late: Final = record("late", 10001.0, user_id="late-user")
@ -117,6 +124,10 @@ async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma,
before: Final = await _session(db, late)
assert before["savings_estimated_actual_spend"] == before["spend"] == 0.17
assert before["saved_spend"] == 0.0
before_daily: Final = await _daily_user(db, late)
assert before_daily["autorouter_estimated_requests"] == 1
assert before_daily["autorouter_estimated_actual_spend"] == before["spend"]
assert before_daily["autorouter_savings_spend"] == 0.0
before_users: Final = await _user_sessions(db, late)
assert set(before_users) == {"late-user"}
assert before_users["late-user"]["savings_estimated_turns"] == 1
@ -126,6 +137,8 @@ async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma,
pending: Final = await _session(db, late)
assert pending["spend"] == 0.34 and pending["savings_estimated_turns"] == 0
assert pending["saved_spend"] == pending["savings_estimated_actual_spend"] == 0.0
pending_daily: Final = await _daily_user(db, late)
assert pending_daily["autorouter_estimated_requests"] == pending_daily["autorouter_estimated_actual_spend"] == 0
pending_users: Final = await _user_sessions(db, late)
assert set(pending_users) == {"late-user", "early-user"}
for user in pending_users.values():
@ -143,6 +156,14 @@ async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma,
assert logs[0]["spend"] == 0.17
assert logs[0]["metadata"]["autorouter_savings_estimate"]["provenance"] == "modeled"
assert after["saved_spend"] == pytest.approx(logs[0]["metadata"]["autorouter_savings"])
after_daily: Final = await _daily_user(db, late)
assert after_daily["autorouter_estimated_requests"] == after["savings_estimated_turns"]
assert after_daily["autorouter_estimated_actual_spend"] == after["savings_estimated_actual_spend"]
for field in (
"api_requests", "autorouter_accounted_requests", "autorouter_requests", "autorouter_llm_spend",
"autorouter_classifier_cost", "autorouter_classifier_cost_recorded_requests",
):
assert after_daily[field] == 0
after_users: Final = await _user_sessions(db, late)
assert after_users["early-user"] == pending_users["early-user"]
for field in (
@ -179,6 +200,10 @@ async def test_commit_ack_loss_and_concurrent_duplicate_delivery_are_idempotent(
session: Final = await _session(db, event)
assert session["turns"] == session["savings_estimated_turns"] == 2
assert session["spend"] == session["savings_estimated_actual_spend"] == 0.34
daily: Final = await _daily_user(db, event)
assert daily["autorouter_estimated_requests"] == session["savings_estimated_turns"]
assert daily["autorouter_estimated_actual_spend"] == session["savings_estimated_actual_spend"]
assert daily["autorouter_accounted_requests"] == daily["autorouter_requests"] == 0
users: Final = await _user_sessions(db, event)
assert set(users) == ({"first-user", "second-user"} if attributed else set())
for user in users.values():
@ -312,3 +337,155 @@ async def test_native_observation_enters_spend_pipeline_once_with_shared_daily_a
assert tag_rows[0]["spend"] == tag_rows[0]["api_requests"] == 0
finally:
await client.db.disconnect()
async def test_old_publisher_transitions_keep_daily_comparison_complete_after_retention(
db: Prisma, record: Callable[..., BaselineAccountingRecord]
) -> None:
event: Final = record("old-publisher")
assert event.daily is not None
assert await _store(db).append(event) == "recorded"
existing_actual: Final = 5.0
other_savings: Final = 3.0
actual: Final = event.pricing.actual_spend
row: Final = {
**event.daily.model_dump(exclude={"targets"}),
"user_id": event.api_key,
"spend": existing_actual + actual,
"api_requests": 2,
"successful_requests": 2,
"autorouter_accounted_requests": 2,
"autorouter_requests": 2,
"autorouter_llm_spend": existing_actual + actual,
"autorouter_classifier_cost_recorded_requests": 2,
"autorouter_estimated_requests": 1,
"autorouter_estimated_actual_spend": existing_actual,
"autorouter_savings_spend": other_savings,
}
table: Final = DAILY_SPEND_TABLES["user"]
statement, values = build_bulk_upsert(table, merge_by_conflict_key(table, (row,)))
await db.execute_raw(statement, *values)
for status, baseline, savings_delta, expected_savings, count, covered_actual in (
("estimated", actual + 0.5, 0.5, other_savings + 0.5, 2, existing_actual + actual),
("estimated", actual, -0.5, other_savings, 2, existing_actual + actual),
("unknown", None, 0.0, other_savings, 1, existing_actual),
("estimated", actual + 0.2, 0.2, other_savings + 0.2, 2, existing_actual + actual),
("estimated", actual + 0.2, 0.0, other_savings + 0.2, 2, existing_actual + actual),
):
publication: Final = BaselinePublication(
comparison_id=event.scope,
comparison_started_at=event.observation.started_at,
status=status,
reason="legacy-publisher-transition",
actual_spend=actual if baseline is not None else None,
baseline_spend=baseline,
)
async with db.tx() as tx:
if savings_delta:
await tx.execute_raw(
'UPDATE "LiteLLM_DailyUserSpend" SET autorouter_savings_spend=autorouter_savings_spend+$1::float8 '
"WHERE api_key=$2 AND user_id=$3",
savings_delta,
event.api_key,
event.api_key,
)
await tx.execute_raw(
'UPDATE "LiteLLM_AutoRouterBaselineObservation" SET publication=$1 WHERE request_id=$2',
publication.model_dump_json(),
event.observation.request_id,
)
daily: Final = await _daily_user(db, event)
assert daily["autorouter_estimated_requests"] == count
assert daily["autorouter_estimated_actual_spend"] == pytest.approx(covered_actual)
assert daily["autorouter_savings_spend"] == pytest.approx(expected_savings)
assert daily["api_requests"] == daily["autorouter_accounted_requests"] == daily["autorouter_requests"] == 2
assert daily["spend"] == daily["autorouter_llm_spend"] == pytest.approx(existing_actual + actual)
assert daily["autorouter_classifier_cost"] == 0
assert daily["autorouter_classifier_cost_recorded_requests"] == 2
shadow: Final = await db.query_raw(
"SELECT publication::jsonb = daily_costs_publication::jsonb AS accounted "
'FROM "LiteLLM_AutoRouterBaselineObservation" WHERE request_id=$1',
event.observation.request_id,
)
assert shadow == [{"accounted": True}]
retained: Final = await _daily_user(db, event)
await db.execute_raw(
'DELETE FROM "LiteLLM_AutoRouterBaselineObservation" WHERE request_id=$1', event.observation.request_id
)
assert await _daily_user(db, event) == retained
@pytest.mark.parametrize("targets", ("missing", "duplicate", "null-and-empty"))
async def test_legacy_zero_publication_repairs_shadow_once_for_normalized_user_targets(
db: Prisma, record: Callable[..., BaselineAccountingRecord], targets: str
) -> None:
source: Final = record("legacy-zero")
assert source.daily is not None
identities: Final = (
() if targets == "missing" else ((None, "") if targets == "null-and-empty" else (source.api_key,) * 2)
)
event: Final = source.model_copy(
update={
"daily": source.daily.model_copy(
update={
"targets": tuple(DailyBaselineTarget(entity="user", entity_id=identity) for identity in identities),
}
)
}
)
publication: Final = BaselinePublication(
comparison_id=event.scope,
comparison_started_at=event.observation.started_at,
status="estimated",
reason="legacy-zero",
actual_spend=event.pricing.actual_spend,
baseline_spend=event.pricing.actual_spend,
).model_dump_json()
await db.execute_raw(
'INSERT INTO "LiteLLM_AutoRouterBaselineObservation" (request_id,scope,started_at,revision,data,publication) '
"VALUES ($1,$2,$3::float8,1,$4,$5)",
event.observation.request_id,
event.scope,
event.observation.started_at,
event.model_dump_json(),
publication,
)
before: Final = await db.query_raw(
'SELECT daily_costs_publication FROM "LiteLLM_AutoRouterBaselineObservation" WHERE request_id=$1',
event.observation.request_id,
)
assert before == [{"daily_costs_publication": None}]
for _ in range(2):
await db.execute_raw(
'UPDATE "LiteLLM_AutoRouterBaselineObservation" SET publication=$1 WHERE request_id=$2',
publication,
event.observation.request_id,
)
rows: Final = await db.query_raw(
"SELECT user_id,autorouter_estimated_requests,autorouter_estimated_actual_spend,autorouter_savings_spend, "
'api_requests,autorouter_requests,spend FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1',
event.api_key,
)
assert rows == (
[
{
"user_id": "" if targets == "null-and-empty" else event.api_key,
"autorouter_estimated_requests": 1,
"autorouter_estimated_actual_spend": event.pricing.actual_spend,
"autorouter_savings_spend": 0,
"api_requests": 0,
"autorouter_requests": 0,
"spend": 0,
}
]
if identities
else []
)
shadow: Final = await db.query_raw(
"SELECT publication::jsonb = daily_costs_publication::jsonb AS accounted "
'FROM "LiteLLM_AutoRouterBaselineObservation" WHERE request_id=$1',
event.observation.request_id,
)
assert shadow == [{"accounted": True}]

View file

@ -1,9 +1,11 @@
"""Idempotent journal migration and primary transactional ownership."""
import asyncio
import json
import os
import time
from collections.abc import AsyncGenerator, Iterator
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager
from datetime import timedelta
from pathlib import Path
@ -101,3 +103,319 @@ async def test_locked_projection_is_bounded_and_cancellation_propagates(
with pytest.raises(asyncio.CancelledError):
await pending
assert await store.project("scope") == "unchanged"
_COVERAGE_MIGRATION: Final = Path(__file__).parents[2] / (
"litellm-proxy-extras/litellm_proxy_extras/migrations/20260923000000_add_daily_autorouter_costs/migration.sql"
)
@pytest.fixture
def _coverage_postgresql() -> Iterator[psycopg.Connection[tuple[object, ...]]]:
base: Final = os.environ["DATABASE_URL"].split("?")[0]
name: Final = f"coverage_{uuid4().hex}"
with psycopg.connect(base, autocommit=True) as admin:
admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(name)))
try:
with psycopg.connect(base, dbname=name) as connection:
yield connection
finally:
admin.execute(sql.SQL("DROP DATABASE {}").format(sql.Identifier(name)))
_COVERAGE_BASE: Final = """
CREATE TABLE "LiteLLM_DailyUserSpend" (
id TEXT PRIMARY KEY, user_id TEXT, date TEXT NOT NULL, api_key TEXT NOT NULL,
model TEXT, custom_llm_provider TEXT, mcp_namespaced_tool_name TEXT, endpoint TEXT,
model_group TEXT, spend DOUBLE PRECISION DEFAULT 0, autorouter_savings_spend DOUBLE PRECISION DEFAULT 0,
updated_at TIMESTAMP,
UNIQUE (user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint)
);
CREATE TABLE "LiteLLM_AutoRouterBaselineObservation" (
request_id TEXT PRIMARY KEY, data TEXT, publication TEXT
);
INSERT INTO "LiteLLM_DailyUserSpend"
(id,user_id,date,api_key,model,custom_llm_provider,mcp_namespaced_tool_name,endpoint,spend,autorouter_savings_spend)
VALUES ('recorded','owner','2026-09-23','key','','','','',10,30)
"""
def _coverage_tables(conn: psycopg.Connection[tuple[object, ...]]) -> None:
conn.execute(_COVERAGE_BASE)
conn.execute(
'INSERT INTO "LiteLLM_AutoRouterBaselineObservation" (request_id,data) VALUES (%s,%s)',
(
"request",
json.dumps(
{
"daily": {
"date": "2026-09-23",
"api_key": "key",
"targets": [{"entity": "user", "entity_id": "owner"}],
}
}
),
),
)
conn.commit()
def _coverage_install(conn: psycopg.Connection[tuple[object, ...]]) -> None:
conn.execute(_COVERAGE_MIGRATION.read_text())
def _coverage_publish(conn: psycopg.Connection[tuple[object, ...]]) -> tuple[object, ...] | None:
conn.execute(
'UPDATE "LiteLLM_AutoRouterBaselineObservation" SET publication=%s WHERE request_id=%s',
(json.dumps({"status": "estimated", "actual_spend": 2, "baseline_spend": 2}), "request"),
)
row: Final = conn.execute(
"SELECT id,spend,autorouter_savings_spend,autorouter_estimated_requests,autorouter_estimated_actual_spend "
'FROM "LiteLLM_DailyUserSpend"'
).fetchone()
conn.commit()
return row
def _coverage_repeat_during_traffic(conn: psycopg.Connection[tuple[object, ...]]) -> None:
with (
psycopg.connect(conn.info.dsn, password=conn.info.password) as traffic,
psycopg.connect(conn.info.dsn, password=conn.info.password, autocommit=True) as installer,
):
traffic.execute('SELECT * FROM "LiteLLM_DailyUserSpend"')
traffic.execute('UPDATE "LiteLLM_AutoRouterBaselineObservation" SET data=data')
with ThreadPoolExecutor(max_workers=1) as pool:
repeated: Final = pool.submit(_coverage_install, installer)
try:
repeated.result(timeout=2)
finally:
installer.cancel()
traffic.rollback()
def test_coverage_install_preserves_history_and_repeat_does_not_wait_for_table_traffic(
_coverage_postgresql: psycopg.Connection[tuple[object, ...]],
) -> None:
_coverage_tables(_coverage_postgresql)
_coverage_install(_coverage_postgresql)
_coverage_repeat_during_traffic(_coverage_postgresql)
assert _coverage_publish(_coverage_postgresql) == ("recorded", 10, 30, 1, 2)
assert _coverage_publish(_coverage_postgresql) == ("recorded", 10, 30, 1, 2)
@pytest.mark.parametrize("damage", ("missing", "disabled", "wrong-function", "wrong-event", "wrong-body"))
def test_coverage_installer_repairs_trigger_before_publishing_without_rewriting_history(
_coverage_postgresql: psycopg.Connection[tuple[object, ...]],
damage: str,
) -> None:
_coverage_tables(_coverage_postgresql)
_coverage_install(_coverage_postgresql)
if damage == "disabled":
_coverage_postgresql.execute(
'ALTER TABLE "LiteLLM_AutoRouterBaselineObservation" '
"DISABLE TRIGGER litellm_update_daily_autorouter_coverage"
)
elif damage == "wrong-body":
_coverage_postgresql.execute(
"CREATE OR REPLACE FUNCTION litellm_update_daily_autorouter_coverage() "
"RETURNS TRIGGER LANGUAGE plpgsql AS 'BEGIN RETURN NEW; END;'"
)
else:
_coverage_postgresql.execute(
'DROP TRIGGER litellm_update_daily_autorouter_coverage ON "LiteLLM_AutoRouterBaselineObservation"'
)
if damage == "wrong-function":
_coverage_postgresql.execute(
"CREATE FUNCTION coverage_noop() RETURNS TRIGGER LANGUAGE plpgsql AS 'BEGIN RETURN NEW; END;'"
)
_coverage_postgresql.execute(
"CREATE TRIGGER litellm_update_daily_autorouter_coverage BEFORE UPDATE OF publication "
'ON "LiteLLM_AutoRouterBaselineObservation" FOR EACH ROW EXECUTE FUNCTION coverage_noop()'
)
elif damage == "wrong-event":
_coverage_postgresql.execute(
"CREATE TRIGGER litellm_update_daily_autorouter_coverage BEFORE UPDATE OF data "
'ON "LiteLLM_AutoRouterBaselineObservation" FOR EACH ROW '
"EXECUTE FUNCTION litellm_update_daily_autorouter_coverage()"
)
_coverage_postgresql.commit()
_coverage_install(_coverage_postgresql)
assert _coverage_publish(_coverage_postgresql) == ("recorded", 10, 30, 1, 2)
@pytest.mark.parametrize("version", (None, 2))
def test_coverage_installer_retains_compatible_unversioned_and_newer_functions(
_coverage_postgresql: psycopg.Connection[tuple[object, ...]],
version: int | None,
) -> None:
_coverage_tables(_coverage_postgresql)
_coverage_install(_coverage_postgresql)
before: Final = _coverage_postgresql.execute(
"SELECT prosrc FROM pg_proc WHERE oid='litellm_update_daily_autorouter_coverage()'::regprocedure"
).fetchone()
assert before is not None and isinstance(before[0], str)
body: Final = before[0] if version is None else "\n" + before[0]
_coverage_postgresql.execute(
sql.SQL(
"CREATE OR REPLACE FUNCTION litellm_update_daily_autorouter_coverage() "
"RETURNS TRIGGER LANGUAGE plpgsql AS {}"
).format(sql.Literal(body))
)
_coverage_postgresql.execute(
sql.SQL("COMMENT ON FUNCTION litellm_update_daily_autorouter_coverage() IS {}").format(
sql.Literal(None if version is None else f"litellm:autorouter_daily_coverage:{version}")
)
)
_coverage_postgresql.commit()
_coverage_repeat_during_traffic(_coverage_postgresql)
after: Final = _coverage_postgresql.execute(
"SELECT prosrc,obj_description(oid,'pg_proc') FROM pg_proc "
"WHERE oid='litellm_update_daily_autorouter_coverage()'::regprocedure"
).fetchone()
assert after == (body, None if version is None else f"litellm:autorouter_daily_coverage:{version}")
assert _coverage_publish(_coverage_postgresql) == ("recorded", 10, 30, 1, 2)
@pytest.mark.parametrize(
"incompatible,reason",
(
(
'ALTER TABLE "LiteLLM_DailyUserSpend" ALTER COLUMN autorouter_estimated_requests TYPE INTEGER',
"incompatible column definitions",
),
(
'ALTER TABLE "LiteLLM_DailyUserSpend" ALTER COLUMN autorouter_estimated_requests SET DEFAULT 99',
"incompatible column definitions",
),
(
"COMMENT ON FUNCTION litellm_update_daily_autorouter_coverage() IS 'litellm:autorouter_daily_coverage:2';"
"ALTER FUNCTION litellm_update_daily_autorouter_coverage() IMMUTABLE",
"Newer daily auto-router coverage is incompatible",
),
(
"COMMENT ON FUNCTION litellm_update_daily_autorouter_coverage() IS 'litellm:autorouter_daily_coverage:2';"
'ALTER TABLE "LiteLLM_DailyUserSpend" DROP COLUMN autorouter_estimated_actual_spend',
"Newer daily auto-router coverage is incompatible",
),
pytest.param(
"COMMENT ON FUNCTION litellm_update_daily_autorouter_coverage() IS 'litellm:autorouter_daily_coverage:2';"
'DROP TRIGGER litellm_update_daily_autorouter_coverage ON "LiteLLM_AutoRouterBaselineObservation";'
"CREATE TRIGGER litellm_update_daily_autorouter_coverage BEFORE UPDATE OF data "
'ON "LiteLLM_AutoRouterBaselineObservation" FOR EACH ROW '
"EXECUTE FUNCTION litellm_update_daily_autorouter_coverage()",
"Newer daily auto-router coverage is incompatible",
id="newer-trigger-event",
),
pytest.param(
"COMMENT ON FUNCTION litellm_update_daily_autorouter_coverage() IS 'litellm:autorouter_daily_coverage:2';"
'ALTER TABLE "LiteLLM_AutoRouterBaselineObservation" '
"DISABLE TRIGGER litellm_update_daily_autorouter_coverage",
"Newer daily auto-router coverage is incompatible",
id="newer-trigger-disabled",
),
pytest.param(
"COMMENT ON FUNCTION litellm_update_daily_autorouter_coverage() IS 'litellm:autorouter_daily_coverage:2';"
'DROP TRIGGER litellm_update_daily_autorouter_coverage ON "LiteLLM_AutoRouterBaselineObservation"',
"Newer daily auto-router coverage is incompatible",
id="newer-trigger-missing",
),
),
)
def test_coverage_installer_rejects_incompatible_state_without_replacing_it(
_coverage_postgresql: psycopg.Connection[tuple[object, ...]],
incompatible: str,
reason: str,
) -> None:
_coverage_tables(_coverage_postgresql)
_coverage_install(_coverage_postgresql)
_coverage_postgresql.execute(incompatible)
state_query: Final = (
"SELECT p.prosrc,p.provolatile,obj_description(p.oid,'pg_proc'),t.oid,pg_get_triggerdef(t.oid),t.tgenabled "
"FROM pg_proc p LEFT JOIN pg_trigger t "
"ON t.tgrelid='\"LiteLLM_AutoRouterBaselineObservation\"'::regclass "
"AND t.tgname='litellm_update_daily_autorouter_coverage' "
"WHERE p.oid='litellm_update_daily_autorouter_coverage()'::regprocedure"
)
before: Final = _coverage_postgresql.execute(state_query).fetchone()
assert before is not None
_coverage_postgresql.commit()
with pytest.raises(psycopg.errors.RaiseException, match=reason):
_coverage_install(_coverage_postgresql)
_coverage_postgresql.rollback()
assert _coverage_postgresql.execute(state_query).fetchone() == before
assert _coverage_postgresql.execute(
'SELECT id,spend,autorouter_savings_spend FROM "LiteLLM_DailyUserSpend"'
).fetchall() == [("recorded", 10, 30)]
def test_coverage_installer_does_not_trust_or_modify_public_schema_homonyms(
_coverage_postgresql: psycopg.Connection[tuple[object, ...]],
) -> None:
_coverage_tables(_coverage_postgresql)
_coverage_install(_coverage_postgresql)
_coverage_postgresql.execute('CREATE SCHEMA "tenant coverage"')
_coverage_postgresql.execute('SET search_path TO "tenant coverage", public')
_coverage_tables(_coverage_postgresql)
_coverage_install(_coverage_postgresql)
assert _coverage_publish(_coverage_postgresql) == ("recorded", 10, 30, 1, 2)
assert _coverage_postgresql.execute(
'SELECT spend,autorouter_savings_spend,autorouter_estimated_requests FROM public."LiteLLM_DailyUserSpend"'
).fetchall() == [(10, 30, 0)]
assert _coverage_postgresql.execute(
'SELECT daily_costs_publication FROM public."LiteLLM_AutoRouterBaselineObservation"'
).fetchall() == [(None,)]
def _coverage_waiting(conn: psycopg.Connection[tuple[object, ...]], pid: int, kind: str) -> None:
deadline: Final = time.monotonic() + 3
while time.monotonic() < deadline:
if conn.execute(
"SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=%s AND locktype=%s AND NOT granted)", (pid, kind)
).fetchone() == (True,):
return
time.sleep(0.01)
pytest.fail(f"Installer did not reach its expected {kind} wait")
def test_waiting_coverage_installer_rechecks_after_peer_finishes_before_taking_table_locks(
_coverage_postgresql: psycopg.Connection[tuple[object, ...]],
) -> None:
_coverage_tables(_coverage_postgresql)
_coverage_postgresql.autocommit = True
info: Final = _coverage_postgresql.info
with (
psycopg.connect(info.dsn, password=info.password) as original_reader,
psycopg.connect(info.dsn, password=info.password) as next_reader,
psycopg.connect(info.dsn, password=info.password, autocommit=True) as first,
psycopg.connect(info.dsn, password=info.password, autocommit=True) as second,
ThreadPoolExecutor(max_workers=3) as pool,
):
original_reader.execute('SELECT * FROM "LiteLLM_DailyUserSpend"')
installing: Final = pool.submit(_coverage_install, first)
try:
_coverage_waiting(_coverage_postgresql, first.info.backend_pid, "relation")
queued_read: Final = pool.submit(next_reader.execute, 'SELECT * FROM "LiteLLM_DailyUserSpend"')
_coverage_waiting(_coverage_postgresql, next_reader.info.backend_pid, "relation")
waiting: Final = pool.submit(_coverage_install, second)
_coverage_waiting(_coverage_postgresql, second.info.backend_pid, "advisory")
original_reader.rollback()
installing.result(timeout=2)
queued_read.result(timeout=2)
waiting.result(timeout=2)
finally:
first.cancel()
second.cancel()
next_reader.cancel()
original_reader.rollback()
next_reader.rollback()
assert _coverage_publish(_coverage_postgresql) == ("recorded", 10, 30, 1, 2)

View file

@ -1,6 +1,7 @@
"""The status line script is copied verbatim to the user's machine, so these drive it the way Claude Code
and Codex do: the documented stdin payload, a transcript on disk, and the proxy behind an injected fetch."""
import hashlib
import io
import json
import os
@ -155,6 +156,23 @@ class TestCredentials:
class TestSessionCache:
def test_old_coverage_only_cache_is_refreshed_before_displaying_totals(self, tmp_path: Path) -> None:
credentials: Final = Credentials("http://p", "sk")
identity: Final = "\n".join((credentials.base_url, credentials.api_key, SESSION_ID))
old_path: Final = tmp_path / hashlib.sha256(identity.encode()).hexdigest()
old_path.write_text(json.dumps({
"fetched_at": 100.0,
"session": {
**RECORDED._asdict(), "spend": 10.0, "baseline_spend": 1.5,
"turns": 3, "savings_estimated_turns": 1, "savings_estimated_actual_spend": 2.0,
},
}))
def fetch(credentials: Credentials, session_id: str) -> Fetched:
return Fetched(RECORDED, True)
assert load_session(credentials, SESSION_ID, tmp_path, fetch, now=lambda: 101.0) == RECORDED
def test_a_definite_answer_is_served_from_the_cache_within_the_ttl(self, tmp_path):
calls = []
@ -342,6 +360,47 @@ class TestRender:
class TestClaudeCodeMode:
@pytest.mark.parametrize("field", ("baseline_spend", "savings_estimated_baseline_spend"))
@pytest.mark.parametrize("invalid_baseline", ("1.5", True, float("nan"), float("inf")))
def test_invalid_baselines_do_not_display_a_cost_comparison(
self, config_dir: Path, field: str, invalid_baseline: object,
) -> None:
session: Final = statusline_script._session_from_payload({
**RECORDED._asdict(), field: invalid_baseline,
})
assert session is None
assert render("m", session, config_dir, use_color=False) == "Routed to: m"
@pytest.mark.parametrize("estimated_turns, saved_spend, delta", (
(0, 30.0, "-75%"), (0, -2.0, "+25%"),
(1, 30.0, "-75%"), (1, 0.0, "0%"), (1, -2.0, "+25%"),
))
def test_historical_and_current_savings_remain_combined_after_cache_refresh(
self, tmp_path: Path, transcript: Path, config_dir: Path, estimated_turns: int,
saved_spend: float, delta: str,
) -> None:
session: Final = statusline_script._session_from_payload(
{
**RECORDED._asdict(),
"spend": 10.0,
"baseline_spend": 10.0 + saved_spend,
"savings_estimated_baseline_spend": 1.5 if estimated_turns else None,
"turns": 3,
"savings_estimated_turns": estimated_turns,
"savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0,
}
)
assert session is not None
def fetch(credentials: Credentials, session_id: str) -> Fetched:
return Fetched(session, True)
first: Final = _run(_payload(transcript), _env(tmp_path, config_dir), fetch)
assert first == _run(_payload(transcript), _env(tmp_path, config_dir), fetch)
assert first.splitlines()[0] == f"Routed to: claude-sonnet-5 {delta} vs Claude Opus 5"
assert "$10.00" in first and f"${10.0 + saved_spend:.2f}" in first
assert "$1.50" not in first and "Savings unavailable" not in first
@pytest.mark.parametrize("estimated_turns", (0, 1))
def test_current_estimates_keep_the_routed_model_and_compare_only_covered_turns(
self, tmp_path: Path, transcript: Path, config_dir: Path, estimated_turns: int

View file

@ -100,6 +100,15 @@ if not earlier_calls and os.environ.get("FAKE_PRISMA_HANG_FIRST"):
grandchild = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)"])
pathlib.Path(os.environ["FAKE_PRISMA_GRANDCHILD_PIDFILE"]).write_text(str(grandchild.pid))
time.sleep(600)
if sys.argv[1:3] == ["db", "execute"]:
if os.environ.get("FAKE_PRISMA_FAIL_DB_EXECUTE"):
sys.exit(1)
if not any(json.loads(call)[:2] == ["db", "execute"] for call in earlier_calls):
first_failure = os.environ.get("FAKE_PRISMA_FAIL_FIRST_DB_EXECUTE")
if first_failure == "timeout":
time.sleep(600)
if first_failure:
sys.exit(1)
sys.exit(0)
"""
@ -142,6 +151,8 @@ def fake_prisma_cli(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Generato
monkeypatch.setenv("FAKE_PRISMA_GRANDCHILD_PIDFILE", str(cli.grandchild_pidfile))
monkeypatch.setenv("LITELLM_PRISMA_COMMAND_TIMEOUT", "1")
monkeypatch.delenv("FAKE_PRISMA_HANG_FIRST", raising=False)
monkeypatch.delenv("FAKE_PRISMA_FAIL_DB_EXECUTE", raising=False)
monkeypatch.delenv("FAKE_PRISMA_FAIL_FIRST_DB_EXECUTE", raising=False)
yield cli
if cli.grandchild_pidfile.exists():
try:

View file

@ -209,6 +209,13 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key():
"prompt_caching_savings_spend": 0,
"gateway_injected_caching_savings_spend": 0,
"autorouter_savings_spend": 0,
"autorouter_accounted_requests": 0,
"autorouter_requests": 0,
"autorouter_llm_spend": 0,
"autorouter_classifier_cost": 0,
"autorouter_classifier_cost_recorded_requests": 0,
"autorouter_estimated_requests": 0,
"autorouter_estimated_actual_spend": 0,
"total_response_time_ms": 0,
"timed_requests": 0,
}
@ -263,6 +270,13 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions(
"prompt_caching_savings_spend": 0,
"gateway_injected_caching_savings_spend": 0,
"autorouter_savings_spend": 0,
"autorouter_accounted_requests": 0,
"autorouter_requests": 0,
"autorouter_llm_spend": 0,
"autorouter_classifier_cost": 0,
"autorouter_classifier_cost_recorded_requests": 0,
"autorouter_estimated_requests": 0,
"autorouter_estimated_actual_spend": 0,
"total_response_time_ms": 0,
"timed_requests": 0,
}
@ -549,7 +563,7 @@ async def test_every_optional_daily_metric_aggregates(daily_spend_update_queue):
# additive metrics may be declared NotRequired[float] for rows queued by a pod
# running the previous release, so unwrap before matching
args = get_args(annotation)
return (args[0] if args else annotation) in (int, float)
return _numeric(args[0]) if args else annotation in (int, float)
numeric_fields = [
name for name, annotation in BaseDailySpendTransaction.__annotations__.items() if _numeric(annotation)
@ -592,3 +606,22 @@ async def test_optional_metric_missing_from_an_older_payload_still_aggregates(
assert updates[0][test_key]["autorouter_savings_spend"] == pytest.approx(0.25)
assert updates[0][test_key]["total_response_time_ms"] == 900
assert updates[0][test_key]["timed_requests"] == 1
@pytest.mark.parametrize("reverse", (False, True))
def test_old_daily_payloads_do_not_gain_router_accounting_coverage(reverse: bool) -> None:
old = {"spend": 1.0, "prompt_tokens": 2, "completion_tokens": 1,
"api_requests": 1, "successful_requests": 1, "failed_requests": 0}
current = {**old, "autorouter_accounted_requests": 1, "autorouter_requests": 1,
"autorouter_llm_spend": 1.0, "autorouter_classifier_cost": 0.2,
"autorouter_classifier_cost_recorded_requests": 1,
"autorouter_estimated_requests": 1, "autorouter_estimated_actual_spend": 1.2}
updates = [{"key": current}, {"key": old}] if reverse else [{"key": old}, {"key": current}]
merged = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions(updates)["key"]
assert merged["api_requests"] == 2
assert merged["autorouter_accounted_requests"] == merged["autorouter_requests"] == 1
assert merged["autorouter_llm_spend"] == 1.0
assert merged["autorouter_classifier_cost"] == 0.2
assert merged["autorouter_classifier_cost_recorded_requests"] == merged["autorouter_estimated_requests"] == 1
assert merged["autorouter_estimated_actual_spend"] == 1.2
assert "autorouter_accounted_requests" not in old

View file

@ -0,0 +1,286 @@
import json
import re
from collections.abc import Mapping
from typing import Final
import psycopg
import pytest
from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
from pytest_postgresql import factories
from litellm.proxy.db.autorouter_daily_spend import AutoRouterDailyCosts
from litellm.proxy.db.autorouter_historical_spend import AUTOROUTER_HISTORICAL_COSTS_SQL
from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
from litellm.proxy.spend_tracking.savings import classifier_cost_from_decision, known_autorouter_savings
_historical_postgresql_proc: Final = factories.postgresql_proc()
_historical_postgresql: Final = factories.postgresql("_historical_postgresql_proc")
_DDL: Final = """
CREATE TABLE "LiteLLM_DailyUserSpend" (
date TEXT, user_id TEXT, api_key TEXT, model TEXT, custom_llm_provider TEXT,
mcp_namespaced_tool_name TEXT, endpoint TEXT,
api_requests BIGINT DEFAULT 0, successful_requests BIGINT DEFAULT 0, failed_requests BIGINT DEFAULT 0,
prompt_tokens BIGINT DEFAULT 0, completion_tokens BIGINT DEFAULT 0, spend FLOAT8 DEFAULT 0,
autorouter_requests BIGINT DEFAULT 0, autorouter_llm_spend FLOAT8 DEFAULT 0,
autorouter_classifier_cost FLOAT8 DEFAULT 0, autorouter_classifier_cost_recorded_requests BIGINT DEFAULT 0,
autorouter_estimated_requests BIGINT DEFAULT 0, autorouter_estimated_actual_spend FLOAT8 DEFAULT 0,
autorouter_savings_spend FLOAT8 DEFAULT 0, autorouter_accounted_requests BIGINT DEFAULT 0
);
CREATE TABLE "LiteLLM_SpendLogs" (
request_id TEXT PRIMARY KEY, "startTime" TIMESTAMP, "user" TEXT, api_key TEXT,
model TEXT, custom_llm_provider TEXT, mcp_namespaced_tool_name TEXT, call_type TEXT,
spend FLOAT8, prompt_tokens BIGINT, completion_tokens BIGINT, status TEXT, metadata JSONB
)
"""
def _seed(conn: psycopg.Connection) -> None:
conn.execute(_DDL)
conn.execute(
"""INSERT INTO "LiteLLM_DailyUserSpend"
(date, user_id, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint,
api_requests, successful_requests, prompt_tokens, completion_tokens, spend, autorouter_savings_spend)
VALUES ('2026-09-22', 'owner', 'key', 'model', NULL, '', '/chat/completions', 2, 2, 55, 10, 5.1, 4)"""
)
decision: Final = {"router_model_name": "router", "classifier_cost": 0.1}
rows: Final = (
("routed", 2.0, 20, 10, {"routing_decision": decision, "autorouter_savings": 4}),
("ordinary", 3.0, 30, 0, {}),
("classifier", 0.1, 5, 0, {"internal_call_origin": "autorouter_classifier", "routing_decision": decision}),
)
for request_id, spend, prompt, completion, metadata in rows:
conn.execute(
"""INSERT INTO "LiteLLM_SpendLogs"
(request_id, "startTime", "user", api_key, model, custom_llm_provider, mcp_namespaced_tool_name,
call_type, spend, prompt_tokens, completion_tokens, status, metadata)
VALUES (%s, '2026-09-22 12:00:00', 'owner', 'key', 'model', '', NULL,
'acompletion', %s, %s, %s, 'success', %s)""",
(request_id, spend, prompt, completion, Jsonb(metadata)),
)
def _read(conn: psycopg.Connection, *, key: str | None = None, user: str | None = None) -> AutoRouterDailyCosts:
query: Final = re.sub(r"\$(\d+)", r"%(p\1)s", AUTOROUTER_HISTORICAL_COSTS_SQL)
with conn.cursor(row_factory=dict_row) as cursor:
cursor.execute(
query, # pyright: ignore[reportArgumentType] # fixed production SQL uses Prisma parameter syntax
{"p1": "2026-09-22", "p2": "2026-09-22", "p3": key, "p4": user, "p5": json.dumps(ROUTE_ENDPOINT_MAPPING)},
)
return AutoRouterDailyCosts.model_validate(cursor.fetchone())
def test_historical_costs_include_no_session_turns_and_classifier_charge_once(
_historical_postgresql: psycopg.Connection,
) -> None:
_seed(_historical_postgresql)
costs: Final = _read(_historical_postgresql, key="key", user="owner")
assert costs == AutoRouterDailyCosts(
requests=1,
llm_spend=2,
classifier_cost=0.1,
classifier_requests=1,
estimated_requests=1,
estimated_actual_spend=2.1,
saved_spend=4,
)
assert costs.recorded_spend == pytest.approx(2.1)
assert costs.baseline_spend(4) == pytest.approx(6.1)
assert _read(_historical_postgresql, key="other").requests == 0
assert _read(_historical_postgresql, user="other").requests == 0
@pytest.mark.parametrize("missing_request", ("ordinary", "classifier", "routed"))
def test_missing_logs_cannot_certify_costs_even_when_saved_savings_match(
_historical_postgresql: psycopg.Connection, missing_request: str
) -> None:
_seed(_historical_postgresql)
_historical_postgresql.execute('DELETE FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (missing_request,))
costs: Final = _read(_historical_postgresql)
assert not costs.complete
assert costs.saved_spend == 4
assert costs.recorded_spend is None
assert costs.baseline_spend(4) is None
def test_missing_zero_cost_request_invalidates_matching_spend_and_token_totals(
_historical_postgresql: psycopg.Connection,
) -> None:
_seed(_historical_postgresql)
_historical_postgresql.execute(
"""UPDATE "LiteLLM_DailyUserSpend" SET spend = 2.1, prompt_tokens = 25;
DELETE FROM "LiteLLM_SpendLogs" WHERE request_id = 'ordinary'"""
)
costs: Final = _read(_historical_postgresql)
assert not costs.complete
assert costs.recorded_spend is None
assert costs.saved_spend == 4
@pytest.mark.parametrize("changed", ("model", "custom_llm_provider", "mcp_namespaced_tool_name", "call_type"))
def test_equal_totals_in_different_daily_identity_do_not_certify_recovery(
_historical_postgresql: psycopg.Connection, changed: str
) -> None:
_seed(_historical_postgresql)
_historical_postgresql.execute(
psycopg.sql.SQL('UPDATE "LiteLLM_SpendLogs" SET {} = %s WHERE request_id = %s').format(
psycopg.sql.Identifier(changed)
),
("different", "ordinary"),
)
costs: Final = _read(_historical_postgresql)
assert not costs.complete
assert costs.recorded_spend is None
@pytest.mark.parametrize(
"metadata,saved,estimated,comparison",
(
({"autorouter_savings": 0}, 0, 1, True),
({"autorouter_savings": -1}, -1, 1, True),
({"autorouter_savings": 0, "autorouter_savings_estimate": {"version": 3, "status": "unknown"}}, 0, 0, True),
({"autorouter_savings": 0, "autorouter_savings_estimate": {"version": 3, "status": "estimated"}}, 0, 1, True),
({}, 4, 0, False),
({"autorouter_savings": True}, 4, 0, False),
({"autorouter_savings": "4"}, 4, 0, False),
({"autorouter_savings": 4, "autorouter_savings_estimate": {}}, 4, 0, False),
(
{"autorouter_savings": 4, "autorouter_savings_estimate": {"version": 1.0, "status": "estimated"}},
4,
0,
False,
),
(
{"autorouter_savings": 4, "autorouter_savings_estimate": {"version": True, "status": "estimated"}},
4,
0,
False,
),
({"autorouter_savings": 4, "autorouter_savings_estimate": {"version": 99, "status": "estimated"}}, 4, 0, False),
),
)
def test_recorded_estimate_provenance_gates_baseline_without_hiding_actual_cost(
_historical_postgresql: psycopg.Connection,
metadata: Mapping[str, object],
saved: float,
estimated: int,
comparison: bool,
) -> None:
_seed(_historical_postgresql)
_historical_postgresql.execute(
'UPDATE "LiteLLM_SpendLogs" SET metadata = %s WHERE request_id = %s',
(Jsonb({"routing_decision": {"router_model_name": "router", "classifier_cost": 0.1}, **metadata}), "routed"),
)
_historical_postgresql.execute('UPDATE "LiteLLM_DailyUserSpend" SET autorouter_savings_spend = %s', (saved,))
costs: Final = _read(_historical_postgresql)
estimate: Final = metadata.get("autorouter_savings_estimate")
recorded: Final = known_autorouter_savings(
model=None,
custom_llm_provider=None,
routing_decision=None,
usage_object=None,
recorded_autorouter_savings=metadata.get("autorouter_savings"),
recorded_autorouter_savings_estimate=estimate if isinstance(estimate, Mapping) else None,
)
assert costs.complete and costs.recorded_spend == pytest.approx(2.1)
assert costs.estimated_requests == estimated == int(recorded is not None)
assert costs.comparison_complete is comparison
assert costs.saved_spend == saved
assert costs.baseline_spend(saved) == (pytest.approx(2.1 + saved) if estimated and comparison else None)
def test_unknown_classifier_preserves_recorded_subtotal_with_partial_coverage(
_historical_postgresql: psycopg.Connection,
) -> None:
_seed(_historical_postgresql)
_historical_postgresql.execute(
"""UPDATE "LiteLLM_SpendLogs" SET metadata = metadata #- '{routing_decision,classifier_cost}'
WHERE request_id = 'routed'"""
)
costs: Final = _read(_historical_postgresql)
assert costs.complete and costs.recorded_llm_spend == 2
assert costs.recorded_classifier_cost is None
assert costs.recorded_spend == 2
assert costs.coverage == "partial"
assert costs.baseline_spend(4) is None
@pytest.mark.parametrize("classifier,expected", ((-0.1, -0.1), (0, 0), (True, None), ("0.1", None)))
def test_classifier_numeric_contract_matches_daily_owner(
_historical_postgresql: psycopg.Connection, classifier: object, expected: float | None
) -> None:
_seed(_historical_postgresql)
_historical_postgresql.execute(
"""UPDATE "LiteLLM_SpendLogs" SET metadata = jsonb_set(metadata, '{routing_decision,classifier_cost}', %s)
WHERE request_id = 'routed'""",
(Jsonb(classifier),),
)
costs: Final = _read(_historical_postgresql)
assert costs.recorded_classifier_cost == expected == classifier_cost_from_decision({"classifier_cost": classifier})
assert costs.classifier_requests == int(expected is not None)
assert costs.recorded_spend == pytest.approx(2 + (expected or 0))
def test_unknown_routed_request_is_counted_in_actual_but_excluded_from_baseline(
_historical_postgresql: psycopg.Connection,
) -> None:
_seed(_historical_postgresql)
_historical_postgresql.execute(
'UPDATE "LiteLLM_SpendLogs" SET metadata = %s WHERE request_id = %s',
(
Jsonb(
{
"routing_decision": {"router_model_name": "router", "classifier_cost": 0},
"autorouter_savings_estimate": {"version": 3, "status": "unknown"},
}
),
"ordinary",
),
)
costs: Final = _read(_historical_postgresql)
assert costs.complete and costs.comparison_complete
assert costs.requests == 2 and costs.estimated_requests == 1
assert costs.recorded_spend == pytest.approx(5.1)
assert costs.baseline_spend(4) == pytest.approx(6.1)
def test_historical_logs_replace_partial_bucket_and_retain_durable_other_days(
_historical_postgresql: psycopg.Connection,
) -> None:
_seed(_historical_postgresql)
_historical_postgresql.execute(
"""UPDATE "LiteLLM_DailyUserSpend" SET autorouter_accounted_requests = 1,
autorouter_requests = 1, autorouter_llm_spend = 1, autorouter_classifier_cost = 0.05,
autorouter_classifier_cost_recorded_requests = 1, autorouter_estimated_requests = 1,
autorouter_estimated_actual_spend = 1.05;
INSERT INTO "LiteLLM_DailyUserSpend"
(date, user_id, api_key, model, endpoint, api_requests, successful_requests,
autorouter_accounted_requests, autorouter_requests, autorouter_llm_spend,
autorouter_classifier_cost, autorouter_classifier_cost_recorded_requests,
autorouter_estimated_requests, autorouter_estimated_actual_spend, autorouter_savings_spend)
VALUES ('2026-09-22', 'owner', 'key', 'durable-model', '/chat/completions', 1, 1, 1, 1, 3, 0.2, 1, 1, 3.2, 2),
('2026-09-21', 'owner', 'key', 'durable-model', '/chat/completions', 1, 1, 1, 1, 90, 1, 1, 1, 91, 9)"""
)
costs: Final = _read(_historical_postgresql)
assert costs.complete and costs.requests == 2
assert costs.recorded_spend == pytest.approx(5.3)
assert costs.saved_spend == 6
assert costs.baseline_spend(6) == pytest.approx(11.3)

View file

@ -3,6 +3,7 @@
import re
from collections.abc import AsyncIterator
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from typing import Final
import pytest
@ -55,20 +56,30 @@ def test_conflict_key_normalizes_every_nullable_key_column(column):
@pytest.mark.parametrize("order", [("null_first"), ("empty_first")])
def test_null_and_empty_provider_merge_into_one_row(order):
@pytest.mark.parametrize("entity", ["user", "tag"])
def test_null_and_empty_provider_merge_into_one_row(order, entity):
"""Two queue entries differing only in NULL versus '' arbitrate to the same row.
Postgres rejects one statement touching a row twice, so they must be folded first.
Asserted under both input orders: a single ordering would prove nothing here."""
null_entry = tag_txn(custom_llm_provider=None, spend=0.25, api_requests=1)
empty_entry = tag_txn(custom_llm_provider="", spend=0.75, api_requests=3)
metrics: Final = {
"autorouter_accounted_requests": 1, "autorouter_requests": 1,
"autorouter_llm_spend": 0.5, "autorouter_classifier_cost": 0.125,
"autorouter_classifier_cost_recorded_requests": 1,
"autorouter_estimated_requests": 1, "autorouter_estimated_actual_spend": 0.625,
}
null_entry = tag_txn(custom_llm_provider=None, spend=0.25, api_requests=1, **metrics)
empty_entry = tag_txn(custom_llm_provider="", spend=0.75, api_requests=3, **metrics)
transactions = (null_entry, empty_entry) if order == "null_first" else (empty_entry, null_entry)
merged = merge_by_conflict_key(TAG_TABLE, transactions)
merged = merge_by_conflict_key(DAILY_SPEND_TABLES[entity], transactions)
assert len(merged) == 1
_, folded = merged[0]
assert folded["spend"] == pytest.approx(1.0)
assert folded["api_requests"] == 4
if entity == "user":
for field, value in metrics.items():
assert folded[field] == value * 2
def test_distinct_keys_are_not_merged_and_are_ordered_deterministically():

View file

@ -2789,6 +2789,7 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent(
payload = {
"request_id": "req-no-compression",
"status": "success",
"user": "test-user",
"startTime": "2026-07-17T00:00:00",
"api_key": "test-key",
@ -2819,6 +2820,8 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent(
assert transaction["prompt_caching_savings_spend"] == 0
assert transaction["spend"] == 0.01
assert transaction["autorouter_savings_spend"] == expected
assert transaction["autorouter_estimated_requests"] == int(expected != 0.0)
assert transaction["autorouter_estimated_actual_spend"] == (0.015 if expected != 0.0 else 0.0)
# ---------------------------------------------------------------------------
@ -3580,6 +3583,104 @@ async def test_daily_transaction_internal_call_keeps_spend_but_not_request_count
assert user_sent["successful_requests"] == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("saved, estimate", [
(0.0, None), (-0.25, None), (0.5, None),
(0.0, {"version": 3, "status": "estimated"}),
(0.5, {"version": 3, "status": "unknown"}),
])
@pytest.mark.parametrize("kind", ("routed", "plain", "failed", "internal", "malformed", "empty"))
async def test_daily_router_costs_account_for_every_external_request_without_a_session(
saved: float, estimate: dict[str, object] | None, kind: str,
) -> None:
writer: Final = DBSpendUpdateWriter()
prisma: Final = MagicMock()
prisma.get_request_status.return_value = "failure" if kind == "failed" else "success"
decision: Final = {"router_model_name": "auto", "classifier_cost": 0.125}
metadata: Final = {
"routing_decision": {"plain": None, "malformed": ["not-a-decision"], "empty": {}}.get(kind, decision),
"internal_call_origin": "autorouter_classifier" if kind == "internal" else None,
"autorouter_savings": saved,
"autorouter_savings_estimate": estimate,
}
transaction: Final = await writer._common_add_spend_log_transaction_to_daily_transaction(
payload={
"user": "request-user", "startTime": "2026-09-23T23:59:59+00:00", "api_key": "hash",
"model": "selected", "model_group": "auto", "custom_llm_provider": "openai",
"call_type": "acompletion", "prompt_tokens": 10, "completion_tokens": 5,
"status": "failure" if kind == "failed" else "success", "spend": 0.5, "metadata": json.dumps(metadata),
},
prisma_client=prisma, type="user",
)
assert transaction is not None
routed: Final = kind == "routed"
estimated: Final = routed and (estimate is None or estimate["status"] == "estimated")
assert transaction["autorouter_accounted_requests"] == transaction["api_requests"] == int(kind != "internal")
assert transaction["autorouter_requests"] == int(routed)
assert transaction["autorouter_llm_spend"] == (0.5 if routed else 0.0)
assert transaction["autorouter_classifier_cost"] == (0.125 if routed else 0.0)
assert transaction["autorouter_classifier_cost_recorded_requests"] == int(routed)
assert transaction["autorouter_estimated_requests"] == int(estimated)
assert transaction["autorouter_estimated_actual_spend"] == (0.625 if estimated else 0.0)
@pytest.mark.asyncio
@pytest.mark.parametrize("status", ("success", None, "failure"))
async def test_daily_router_costs_use_status_normalized_from_request_metadata(status: str | None) -> None:
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.proxy.utils import PrismaClient
started_at: Final = datetime(2026, 9, 23, tzinfo=timezone.utc)
metadata: Final = {
"user_api_key": "test-hash", "user_api_key_user_id": "u", "model_group": "auto",
"routing_decision": {"router_model_name": "auto", "classifier_cost": 0.125},
**({"status": status} if status is not None else {}),
}
payload: Final = get_logging_payload(
kwargs={
"model": "synthetic-target", "call_type": "acompletion", "response_cost": 0.5,
"litellm_params": {"metadata": metadata, "custom_llm_provider": "openai"},
},
response_obj={}, start_time=started_at, end_time=started_at,
)
prisma: Final = PrismaClient.__new__(PrismaClient)
transaction: Final = await DBSpendUpdateWriter()._common_add_spend_log_transaction_to_daily_transaction(
payload, prisma, "user",
)
assert transaction is not None
assert payload["status"] == ("failure" if status == "failure" else "success")
assert transaction["autorouter_requests"] == int(status != "failure")
assert transaction["autorouter_llm_spend"] == (0.0 if status == "failure" else 0.5)
assert transaction["autorouter_classifier_cost"] == (0.0 if status == "failure" else 0.125)
@pytest.mark.asyncio
@pytest.mark.parametrize("classifier_cost, recorded", [(None, 0), (True, 0), (float("inf"), 0), (0.0, 1)])
async def test_daily_router_classifier_zero_is_known_only_when_recorded(
classifier_cost: object, recorded: int,
) -> None:
prisma: Final = MagicMock()
prisma.get_request_status.return_value = "success"
transaction: Final = await DBSpendUpdateWriter()._common_add_spend_log_transaction_to_daily_transaction(
payload={
"user": "u", "startTime": "2026-09-23T00:00:00", "api_key": "hash", "model": "m",
"model_group": "auto", "custom_llm_provider": "openai", "call_type": "acompletion",
"prompt_tokens": 1, "completion_tokens": 1, "spend": 1.0, "status": "success",
"metadata": json.dumps({
"routing_decision": {"router_model_name": "auto", "classifier_cost": classifier_cost},
"autorouter_savings": 0.0,
}),
}, prisma_client=prisma, type="user",
)
assert transaction is not None
assert transaction["autorouter_requests"] == 1
assert transaction["autorouter_classifier_cost_recorded_requests"] == recorded
assert transaction["autorouter_classifier_cost"] == 0.0
assert transaction["autorouter_estimated_requests"] == 1
assert transaction["autorouter_estimated_actual_spend"] == 1.0
def _response_time_payload(request_duration_ms: object, metadata: dict | None = None) -> dict:
return {
"request_id": "req-timed-1",

View file

@ -3,6 +3,7 @@ import os
import signal
import sys
import urllib.parse
from typing import TYPE_CHECKING
from unittest.mock import AsyncMock, MagicMock, Mock, patch
import pytest
@ -12,6 +13,9 @@ from fastapi.testclient import TestClient
from litellm.proxy.db.prisma_client import PrismaManager, PrismaWrapper, should_update_prisma_schema
if TYPE_CHECKING:
from tests.test_litellm.proxy.db.conftest import FakePrismaCli
@pytest.fixture(autouse=True)
def mock_prisma_binary():
@ -213,7 +217,10 @@ def test_db_push_applies_replica_identity_full_when_requested(monkeypatch, fake_
assert PrismaManager.setup_database(use_migrate=False) is True
assert fake_prisma_cli.calls == [DB_PUSH_ARGV]
assert len(fake_prisma_cli.calls) == 2
assert fake_prisma_cli.calls[0] == DB_PUSH_ARGV
assert fake_prisma_cli.calls[1][:2] == ["db", "execute"]
assert fake_prisma_cli.calls[1][-1].endswith("20260923000000_add_daily_autorouter_costs/migration.sql")
assert applied == [True]
@ -246,7 +253,47 @@ def test_db_push_proceeds_when_spend_logs_is_not_partitioned(monkeypatch, fake_p
)
assert PrismaManager.setup_database(use_migrate=False) is True
assert fake_prisma_cli.calls == [DB_PUSH_ARGV]
assert len(fake_prisma_cli.calls) == 2
assert fake_prisma_cli.calls[0] == DB_PUSH_ARGV
assert fake_prisma_cli.calls[1][:2] == ["db", "execute"]
@pytest.mark.parametrize("use_migrate", [False, True])
def test_db_push_cannot_succeed_without_daily_coverage_owner(
fake_prisma_cli: "FakePrismaCli",
unset_database_url: None,
monkeypatch: pytest.MonkeyPatch,
use_migrate: bool,
) -> None:
monkeypatch.setenv("FAKE_PRISMA_FAIL_DB_EXECUTE", "1")
with patch("litellm_proxy_extras.utils.time.sleep"), pytest.raises(RuntimeError, match="daily auto-router coverage"):
PrismaManager.setup_database(use_migrate=use_migrate)
assert [call[:2] for call in fake_prisma_cli.calls if call != ["--version"]] == [
["migrate", "deploy"] if use_migrate else ["db", "push"],
*([["db", "execute"]] * 4),
]
@pytest.mark.parametrize("use_migrate", [False, True])
@pytest.mark.parametrize("failure", ["exit", "timeout"])
def test_database_setup_retries_transient_daily_coverage_install(
fake_prisma_cli: "FakePrismaCli",
unset_database_url: None,
monkeypatch: pytest.MonkeyPatch,
use_migrate: bool,
failure: str,
) -> None:
monkeypatch.setenv("FAKE_PRISMA_FAIL_FIRST_DB_EXECUTE", failure)
with patch("litellm_proxy_extras.utils.time.sleep"):
assert PrismaManager.setup_database(use_migrate=use_migrate) is True
assert [call[:2] for call in fake_prisma_cli.calls if call != ["--version"]] == [
["migrate", "deploy"] if use_migrate else ["db", "push"],
["db", "execute"], ["db", "execute"],
]
def _entra_jwt(expires_in_seconds: int) -> str:
@ -439,7 +486,9 @@ def test_db_push_timeout_takes_its_process_tree_with_it(fake_prisma_cli, unset_d
monkeypatch.setenv("FAKE_PRISMA_HANG_FIRST", "1")
assert PrismaManager.setup_database(use_migrate=False) is True
assert fake_prisma_cli.calls == [DB_PUSH_ARGV, DB_PUSH_ARGV]
assert fake_prisma_cli.calls[:2] == [DB_PUSH_ARGV, DB_PUSH_ARGV]
assert len(fake_prisma_cli.calls) == 3
assert fake_prisma_cli.calls[2][:2] == ["db", "execute"]
assert fake_prisma_cli.grandchild_is_gone(within_seconds=5)

View file

@ -2,7 +2,9 @@
Unit tests for auto router management endpoints
"""
import json
from collections.abc import Mapping, Sequence
from datetime import date
from functools import partial
from pathlib import Path
from types import SimpleNamespace
@ -25,6 +27,8 @@ from litellm.proxy._types import (
ProxyException,
UserAPIKeyAuth,
)
from litellm.proxy.db.autorouter_daily_spend import AUTOROUTER_DAILY_COSTS_SQL, AutoRouterDailyCosts
from litellm.proxy.db.autorouter_historical_spend import AUTOROUTER_HISTORICAL_COSTS_SQL
from litellm.proxy.management_endpoints import auto_router_endpoints
from litellm.proxy.management_endpoints.auto_router_endpoints import (
preview_auto_router_routing,
@ -40,6 +44,12 @@ from litellm.types.management_endpoints.auto_router_endpoints import (
AutoRouterBenchmarksResponse,
AutoRouterRoutingTestRequest,
)
from litellm.types.proxy.management_endpoints.common_daily_activity import (
DailySpendData,
DailySpendMetadata,
SpendAnalyticsPaginatedResponse,
SpendMetrics,
)
from litellm.types.router import Deployment
from litellm.types.utils import Choices, Message, ModelResponse
@ -629,18 +639,39 @@ class TestAutoRouterBenchmarks:
monkeypatch.setattr(proxy_server, "llm_router", None)
@pytest.fixture(autouse=True)
def _daily_savings(self, monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy.management_endpoints import common_daily_activity
monkeypatch.setattr(
common_daily_activity,
"get_daily_activity_aggregated",
AsyncMock(return_value=self._daily_result(30.0)),
)
@staticmethod
def _daily_result(saved: float, *, has_rows: bool = True) -> SpendAnalyticsPaginatedResponse:
return SpendAnalyticsPaginatedResponse(
results=[DailySpendData(date=date(2026, 7, 15), metrics=SpendMetrics(autorouter_savings_spend=saved))]
if has_rows else [],
metadata=DailySpendMetadata(total_autorouter_savings_spend=saved),
)
@staticmethod
async def _benchmarks(
monkeypatch: pytest.MonkeyPatch,
rows: Sequence[Mapping[str, object]],
model_list: Sequence[object],
api_key: str | None = None,
costs: AutoRouterDailyCosts | None = None,
) -> AutoRouterBenchmarksResponse:
from litellm.proxy import proxy_server
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks
class _DB:
async def query_raw(self, sql: str, *params: object):
if sql in (AUTOROUTER_DAILY_COSTS_SQL, AUTOROUTER_HISTORICAL_COSTS_SQL):
return [(costs or AutoRouterDailyCosts(complete=False)).model_dump()]
return rows
monkeypatch.setattr(proxy_server, "prisma_client", type("P", (), {"db": _DB()})())
@ -719,24 +750,208 @@ class TestAutoRouterBenchmarks:
assert totals.saved_pct == -100.0
assert totals.classifier_cost == 0.4
@pytest.mark.parametrize("estimated_turns", [0, 4])
def test_savings_compare_only_the_current_estimated_cohort(self, estimated_turns: int) -> None:
from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals
@pytest.mark.asyncio
@pytest.mark.parametrize("estimated_turns,saved_spend,baseline_spend,saved_pct", [
(0, 30.0, 40.0, 75.0), (4, 30.0, 40.0, 75.0),
(0, 0.0, None, None), (4, 0.0, 10.0, 0.0), (0, -2.0, 8.0, -25.0),
])
async def test_session_groups_preserve_history_and_only_compare_known_savings(
self, monkeypatch: pytest.MonkeyPatch, estimated_turns: int, saved_spend: float,
baseline_spend: float | None, saved_pct: float | None,
) -> None:
row: Final = self.ROW.model_copy(
update={
"saved_spend": saved_spend,
"savings_estimated_turns": estimated_turns,
"savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0,
"savings_estimated_saved_spend": -0.5 if estimated_turns else 0.0,
}
)
totals: Final = _benchmark_totals(row)
assert totals.spend == 10.0
assert totals.savings_estimated_turns == estimated_turns
assert totals.saved_spend == (-0.5 if estimated_turns else None)
assert totals.baseline_spend == (1.5 if estimated_turns else None)
assert totals.saved_pct == (pytest.approx(-33.3) if estimated_turns else None)
assert totals.saved_per_session is None
response: Final = await self._benchmarks(monkeypatch, rows=[row.model_dump()], model_list=[])
group: Final = response.groups[0]
assert group.spend == 10.0
assert group.saved_spend == saved_spend
assert group.baseline_spend == baseline_spend
assert group.saved_pct == saved_pct
assert group.saved_per_session == saved_spend / row.sessions
@pytest.mark.asyncio
async def test_tracked_session_groups_keep_historical_and_new_estimates_once(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
historical: Final = self.ROW.model_copy(
update={
"savings_estimated_turns": 0,
"savings_estimated_actual_spend": 0.0,
"savings_estimated_saved_spend": 0.0,
}
)
current: Final = self.ROW.model_copy(
update={
"router_name": "current-auto",
"sessions": 1,
"turns": 4,
"spend": 2.0,
"saved_spend": -0.5,
"savings_estimated_turns": 4,
"savings_estimated_actual_spend": 2.0,
"savings_estimated_saved_spend": -0.5,
}
)
response: Final = await self._benchmarks(
monkeypatch, rows=[historical.model_dump(), current.model_dump()], model_list=[]
)
assert [group.saved_spend for group in response.groups] == [30.0, -0.5]
assert response.groups[1].savings_estimated_turns == 4
@pytest.mark.asyncio
@pytest.mark.parametrize("api_key,user_id", [(None, None), ("key-hash", None), (None, "selected-user"),
("key-hash", "selected-user")])
@pytest.mark.parametrize(
"daily_saved,has_daily_rows,has_sessions",
[(6.5, True, True), (-0.5, True, True), (0.0, True, True), (30.0, True, True),
(2.0, True, False), (0.0, False, True), (0.0, False, False)],
ids=["historical-and-new-plus-no-session", "signed-loss", "recorded-zero", "matching-is-not-proof",
"no-session-history", "out-of-window-session-only", "empty"],
)
async def test_total_uses_overall_request_date_savings_with_identical_filters(
self, monkeypatch: pytest.MonkeyPatch, api_key: str | None, user_id: str | None,
daily_saved: float, has_daily_rows: bool, has_sessions: bool,
) -> None:
from litellm.proxy.management_endpoints import common_daily_activity
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks
daily: Final = AsyncMock(return_value=self._daily_result(daily_saved, has_rows=has_daily_rows))
query: Final = AsyncMock(side_effect=[
[self.ROW.model_dump()] if has_sessions else [],
[AutoRouterDailyCosts(complete=not has_daily_rows, saved_spend=daily_saved).model_dump()],
[AutoRouterDailyCosts(complete=not has_daily_rows, saved_spend=daily_saved).model_dump()],
])
prisma: Final = SimpleNamespace(db=SimpleNamespace(query_raw=query))
monkeypatch.setattr(common_daily_activity, "get_daily_activity_aggregated", daily)
monkeypatch.setattr(proxy_server, "prisma_client", prisma)
response: Final = await get_auto_router_benchmarks(
user_api_key_dict=ADMIN, start_date="2026-07-01", end_date="2026-08-01",
api_key=api_key, user_id=user_id,
)
daily.assert_awaited_once_with(
prisma_client=prisma, table_name="litellm_dailyuserspend", entity_id_field="user_id",
entity_id=user_id, entity_metadata_field=None, start_date="2026-07-01", end_date="2026-08-01",
model=None, api_key=api_key,
)
assert query.call_args_list[0].args[1:] == ("2026-07-01T00:00:00", "2026-08-02T00:00:00", api_key, user_id)
assert query.call_args_list[1].args[1:] == ("2026-07-01", "2026-08-01", api_key, user_id)
assert response.totals.saved_spend == daily_saved
assert response.totals.sessions == (4 if has_sessions else 0)
assert response.totals.saved_per_session == (7.5 if has_sessions else 0.0)
assert [group.saved_spend for group in response.groups] == ([30.0] if has_sessions else [])
assert (
response.totals.spend, response.totals.classifier_cost, response.totals.baseline_spend,
response.totals.saved_pct,
) == ((None,) * 4 if has_daily_rows else (0.0,) * 4)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"complete, classifier_requests, estimated, saved, recorded_saved, expected_spend, baseline, pct, coverage",
[
(True, 3, 2, 30.0, 30.0, 10.0, 32.5, 92.3, "complete"),
(True, 3, 0, 0.0, 0.0, 10.0, None, None, "complete"),
(True, 2, 2, 30.0, 30.0, 10.0, None, None, "partial"),
(False, 3, 2, 30.0, 30.0, 10.0, None, None, "partial"),
(True, 3, 2, 30.0, 29.0, 10.0, None, None, "complete"),
(True, 3, 2, -0.5, -0.5, 10.0, 2.0, -25.0, "complete"),
],
ids=["covered-subset", "unknown-only", "missing-classifier", "partial-history", "pending-savings", "loss"],
)
async def test_daily_costs_preserve_actual_charges_and_compare_only_matching_estimates(
self, monkeypatch: pytest.MonkeyPatch, complete: bool, classifier_requests: int,
estimated: int, saved: float, recorded_saved: float, expected_spend: float | None,
baseline: float | None, pct: float | None, coverage: str,
) -> None:
from litellm.proxy.management_endpoints import common_daily_activity
monkeypatch.setattr(common_daily_activity, "get_daily_activity_aggregated",
AsyncMock(return_value=self._daily_result(saved)))
costs: Final = AutoRouterDailyCosts(
requests=3, llm_spend=9.7, classifier_cost=0.3, classifier_requests=classifier_requests,
estimated_requests=estimated, estimated_actual_spend=2.5 if estimated else 0.0,
saved_spend=recorded_saved, complete=complete,
)
response: Final = await self._benchmarks(monkeypatch, rows=[], model_list=[], costs=costs)
totals: Final = response.totals
assert totals.saved_spend == saved
assert totals.spend == expected_spend
assert totals.llm_spend == 9.7
assert totals.classifier_cost == (0.3 if classifier_requests == 3 else None)
assert totals.baseline_spend == baseline
assert totals.saved_pct == pct
assert totals.cost_coverage == coverage
assert totals.cost_requests == (3 if complete else None)
assert totals.savings_estimated_turns == estimated
assert totals.savings_estimated_actual_spend == (2.5 if estimated else 0.0)
assert totals.sessions == totals.turns == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("recovery", ["complete", "partial", "empty", "failure", "invalid", "not-needed"])
async def test_historical_cost_recovery_preserves_durable_savings_and_request_scope(
self, monkeypatch: pytest.MonkeyPatch, recovery: str,
) -> None:
from litellm.proxy.management_endpoints import common_daily_activity
from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
recovered: Final = AutoRouterDailyCosts(
requests=3, llm_spend=4.5, classifier_cost=0.5, classifier_requests=3,
estimated_requests=2, estimated_actual_spend=3.0, saved_spend=6.5,
complete=recovery != "partial",
)
recorded: Final = recovered if recovery == "not-needed" else AutoRouterDailyCosts(
requests=1, llm_spend=2.0, classifier_cost=0.25, classifier_requests=1,
estimated_requests=1, estimated_actual_spend=2.25, saved_spend=1.5, complete=False,
)
recovery_result: Final = {
"empty": [], "failure": RuntimeError("retained logs unavailable"), "invalid": [{"unexpected": True}],
}.get(recovery, [recovered.model_dump()])
query: Final = AsyncMock(side_effect=[[self.ROW.model_dump()], [recorded.model_dump()], recovery_result])
transaction: Final = SimpleNamespace(query_raw=query, execute_raw=AsyncMock())
reader: Final = MagicMock()
reader.tx.return_value.__aenter__.return_value = transaction
monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(
db=SimpleNamespace(query_raw=query), read_db=reader,
))
monkeypatch.setattr(common_daily_activity, "get_daily_activity_aggregated",
AsyncMock(return_value=self._daily_result(6.5)))
response: Final = await auto_router_endpoints.get_auto_router_benchmarks(
user_api_key_dict=ADMIN, start_date="2026-07-01", end_date="2026-08-01",
api_key="selected-key", user_id="selected-user",
)
totals: Final = response.totals
assert totals.saved_spend == 6.5
assert response.groups[0].saved_spend == self.ROW.saved_spend
fallback: Final = recovery in ("empty", "failure", "invalid")
assert (totals.spend, totals.llm_spend, totals.classifier_cost) == (
(2.25, 2.0, 0.25) if fallback else (5.0, 4.5, 0.5)
)
comparable: Final = recovery in ("complete", "not-needed")
assert (totals.baseline_spend, totals.saved_pct) == ((9.5, 68.4) if comparable else (None, None))
assert totals.cost_coverage == ("complete" if comparable else "partial")
assert query.await_count == (2 if recovery == "not-needed" else 3)
if recovery != "not-needed":
sql, start, end, key, user, endpoints = query.call_args.args
assert sql == AUTOROUTER_HISTORICAL_COSTS_SQL
assert (start, end, key, user) == ("2026-07-01", "2026-08-01", "selected-key", "selected-user")
assert json.loads(endpoints) == ROUTE_ENDPOINT_MAPPING
@pytest.mark.asyncio
async def test_daily_savings_failure_never_falls_back_to_session_savings(self, monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy.management_endpoints import common_daily_activity
monkeypatch.setattr(
common_daily_activity, "get_daily_activity_aggregated",
AsyncMock(side_effect=HTTPException(status_code=500, detail="Daily savings unavailable")),
)
with pytest.raises(HTTPException, match="Daily savings unavailable"):
await self._benchmarks(monkeypatch, rows=[self.ROW.model_dump()], model_list=[])
def test_an_empty_window_folds_to_zeros(self):
from litellm.proxy.management_endpoints.auto_router_endpoints import (
@ -854,6 +1069,8 @@ class TestAutoRouterBenchmarks:
class _DB:
async def query_raw(self, sql: str, *params: object):
if sql in (AUTOROUTER_DAILY_COSTS_SQL, AUTOROUTER_HISTORICAL_COSTS_SQL):
return [AutoRouterDailyCosts(complete=False).model_dump()]
captured["sql"] = sql
captured["params"] = params
return [TestAutoRouterBenchmarks.ROW.model_dump()]
@ -870,9 +1087,12 @@ class TestAutoRouterBenchmarks:
assert captured["params"] == ("2026-07-01T00:00:00", "2026-08-02T00:00:00", "key-hash", user_id)
assert response.routers_in_scope == 1
assert response.groups[0].router_name == "live-auto"
assert response.groups[0].saved_pct == response.totals.saved_pct == 75.0
assert response.groups[0].classifier_cost == response.totals.classifier_cost == 0.4
assert response.totals.spend - response.totals.classifier_cost == pytest.approx(9.6)
assert response.groups[0].saved_pct == 75.0
assert response.groups[0].classifier_cost == 0.4
assert response.totals.saved_spend == 30.0
assert response.totals.saved_pct is None
assert response.totals.classifier_cost is None
assert response.totals.spend is None
@pytest.mark.asyncio
@pytest.mark.parametrize("recorded_turns", [0, 3, 10])
@ -899,12 +1119,12 @@ class TestAutoRouterBenchmarks:
wire: Final = response.model_dump()
assert wire["groups"][0]["classifier_cost"] == 0.4
assert wire["groups"][1]["classifier_cost"] == (pytest.approx(0.2) if recorded_turns == 10 else None)
assert wire["totals"]["classifier_cost"] == (pytest.approx(0.6) if recorded_turns == 10 else None)
assert wire["totals"]["classifier_cost"] is None
assert response.totals.turns == 50
assert response.totals.spend == 12.0
assert response.totals.saved_spend == 29.5
assert response.totals.baseline_spend == 41.5
assert response.totals.saved_pct == 71.1
assert response.totals.spend is None
assert response.totals.saved_spend == 30.0
assert response.totals.baseline_spend is None
assert response.totals.saved_pct is None
assert response.totals.saved_per_session == 5.9
@pytest.mark.asyncio
@ -919,6 +1139,8 @@ class TestAutoRouterBenchmarks:
class _DB:
async def query_raw(self, sql: str, *params: object):
if sql in (AUTOROUTER_DAILY_COSTS_SQL, AUTOROUTER_HISTORICAL_COSTS_SQL):
return [AutoRouterDailyCosts(complete=False).model_dump()]
return [{**TestAutoRouterBenchmarks.ROW.model_dump(), "tier_turns": wire_value}]
monkeypatch.setattr(proxy_server, "prisma_client", type("P", (), {"db": _DB()})())
@ -953,7 +1175,7 @@ class TestAutoRouterBenchmarks:
assert len(response.groups) == 3
assert response.routers_in_scope == 3
assert by_name["live-auto"].spend == 10.0
assert response.totals.spend == 10.0
assert response.totals.spend is None
assert response.totals.sessions == 4
for name in ("idle-from-config", "idle-from-db"):
idle = by_name[name]
@ -1100,13 +1322,17 @@ class TestAutoRouterSession:
@pytest.mark.asyncio
@pytest.mark.parametrize(
"turns, estimated", [(3, True), (10, True), (10, False)], ids=["full", "partial", "legacy"]
"turns, estimated, saved_spend",
[(3, True, 0.24), (10, True, 0.24), (10, False, 0.24), (10, False, 0.0),
(10, True, -0.24), (10, False, -0.24), (3, True, 0.0)],
ids=["full", "partial", "legacy", "unknown", "negative", "legacy-negative", "estimated-zero"],
)
async def test_a_key_reads_its_own_session_with_the_baseline_its_turns_were_priced_against(
self,
monkeypatch: pytest.MonkeyPatch,
turns: int,
estimated: bool,
saved_spend: float,
) -> None:
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session
@ -1115,10 +1341,12 @@ class TestAutoRouterSession:
key: value for key, value in self.ROW.items() if estimated or not key.startswith("savings_estimated_")
}
spend: Final = 0.14 if turns == 3 else 10.0
if estimated and turns != 3:
row["savings_estimated_saved_spend"] = -0.04
if estimated:
row["savings_estimated_saved_spend"] = saved_spend if turns == 3 else -0.04
self._rig(
monkeypatch, [{**row, "api_key": caller.api_key, "session_id": "sess-1", "turns": turns, "spend": spend}]
monkeypatch,
[{**row, "api_key": caller.api_key, "session_id": "sess-1", "turns": turns, "spend": spend,
"saved_spend": saved_spend}],
)
response = await get_auto_router_session(user_api_key_dict=caller, session_id="sess-1")
assert response.model_dump() == {
@ -1128,15 +1356,42 @@ class TestAutoRouterSession:
"turns": turns,
"last_model": "anthropic/claude-sonnet-5",
"spend": spend,
"saved_spend": (0.24 if turns == 3 else -0.04) if estimated else None,
"saved_spend": saved_spend,
"savings_estimated_turns": 3 if estimated else 0,
"savings_estimated_actual_spend": 0.14 if estimated else 0.0,
"baseline_spend": pytest.approx(0.38) if turns == 3 else None,
"savings_estimated_baseline_spend": pytest.approx(0.38 if turns == 3 else 0.1) if estimated else None,
"baseline_model": "anthropic/claude-opus-5" if estimated else None,
"baseline_models": {"anthropic/claude-opus-5": 3} if estimated else {},
"baseline_spend": pytest.approx(spend + saved_spend) if estimated or saved_spend != 0 else None,
"savings_estimated_baseline_spend": pytest.approx(spend + saved_spend if turns == 3 else 0.1)
if estimated else None,
"baseline_model": "anthropic/claude-opus-5",
"baseline_models": {"anthropic/claude-opus-5": 3},
}
@pytest.mark.asyncio
async def test_unknown_only_session_keeps_savings_unavailable_in_the_cli(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path,
) -> None:
from litellm.proxy.client.cli.commands.statusline_script import _session_from_payload, render
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session
caller: Final = UserAPIKeyAuth(api_key="sk-caller")
self._rig(monkeypatch, [{
**self.ROW, "api_key": caller.api_key, "session_id": "unknown-only",
"saved_spend": 0.0, "savings_estimated_turns": 0,
"savings_estimated_actual_spend": 0.0, "savings_estimated_saved_spend": 0.0,
"savings_estimated_baseline_models": {},
}])
response: Final = await get_auto_router_session(user_api_key_dict=caller, session_id="unknown-only")
assert response.turns == 3 and response.savings_estimated_turns == 0
assert response.baseline_model == "anthropic/claude-opus-5"
assert response.baseline_spend is None
assert response.savings_estimated_baseline_spend is None
session: Final = _session_from_payload(response.model_dump())
assert session is not None
assert render("claude-sonnet-5", session, tmp_path, use_color=False) == (
"Routed to: claude-sonnet-5 · Savings unavailable"
)
@pytest.mark.asyncio
async def test_another_keys_session_is_a_404_even_for_an_admin(self, monkeypatch: pytest.MonkeyPatch):
# The scope is the caller's own key hash, exactly what the spend writer keyed the row under;
@ -1168,14 +1423,13 @@ class TestAutoRouterSession:
assert response.router_name == "new-auto"
@pytest.mark.asyncio
async def test_a_reconfigured_router_keeps_the_label_the_money_was_priced_against(
async def test_a_reconfigured_router_retains_historical_baseline_labels(
self, monkeypatch: pytest.MonkeyPatch
):
# The proxy's router now prices against a different baseline, but the row's money was priced
# against opus for two of three turns, and the label says so; the full split is on the response.
) -> None:
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session
priced = {"anthropic/claude-opus-5": 2, "anthropic/claude-sonnet-5": 1}
priced: Final = {"anthropic/claude-opus-5": 2, "anthropic/claude-sonnet-5": 1}
historical: Final = {"old-baseline": 100, **priced}
self._rig(
monkeypatch,
[
@ -1183,14 +1437,14 @@ class TestAutoRouterSession:
**self.ROW,
"api_key": ADMIN.api_key,
"session_id": "s",
"baseline_models": {"old-baseline": 100},
"baseline_models": historical,
"savings_estimated_baseline_models": priced,
}
],
)
response = await get_auto_router_session(user_api_key_dict=ADMIN, session_id="s")
assert response.baseline_model == "anthropic/claude-opus-5"
assert response.baseline_models == priced
response: Final = await get_auto_router_session(user_api_key_dict=ADMIN, session_id="s")
assert response.baseline_model == "old-baseline"
assert response.baseline_models == historical
@pytest.mark.asyncio
async def test_an_oversized_client_session_id_is_bounded_like_the_writer_bounded_it(

View file

@ -1477,6 +1477,13 @@ _DAILY_USER_SPEND_DDL: Final = """
prompt_caching_savings_spend DOUBLE PRECISION DEFAULT 0,
gateway_injected_caching_savings_spend DOUBLE PRECISION DEFAULT 0,
autorouter_savings_spend DOUBLE PRECISION DEFAULT 0,
autorouter_accounted_requests BIGINT NOT NULL DEFAULT 0,
autorouter_requests BIGINT NOT NULL DEFAULT 0,
autorouter_llm_spend DOUBLE PRECISION NOT NULL DEFAULT 0,
autorouter_classifier_cost DOUBLE PRECISION NOT NULL DEFAULT 0,
autorouter_classifier_cost_recorded_requests BIGINT NOT NULL DEFAULT 0,
autorouter_estimated_requests BIGINT NOT NULL DEFAULT 0,
autorouter_estimated_actual_spend DOUBLE PRECISION NOT NULL DEFAULT 0,
spend DOUBLE PRECISION DEFAULT 0,
api_requests BIGINT DEFAULT 0,
successful_requests BIGINT DEFAULT 0,

View file

@ -385,6 +385,13 @@ _DAILY_USER_SPEND_DDL: Final = """
prompt_caching_savings_spend DOUBLE PRECISION DEFAULT 0,
gateway_injected_caching_savings_spend DOUBLE PRECISION DEFAULT 0,
autorouter_savings_spend DOUBLE PRECISION DEFAULT 0,
autorouter_accounted_requests BIGINT NOT NULL DEFAULT 0,
autorouter_requests BIGINT NOT NULL DEFAULT 0,
autorouter_llm_spend DOUBLE PRECISION NOT NULL DEFAULT 0,
autorouter_classifier_cost DOUBLE PRECISION NOT NULL DEFAULT 0,
autorouter_classifier_cost_recorded_requests BIGINT NOT NULL DEFAULT 0,
autorouter_estimated_requests BIGINT NOT NULL DEFAULT 0,
autorouter_estimated_actual_spend DOUBLE PRECISION NOT NULL DEFAULT 0,
spend DOUBLE PRECISION DEFAULT 0,
api_requests BIGINT DEFAULT 0,
successful_requests BIGINT DEFAULT 0,

View file

@ -19,6 +19,30 @@ from litellm.types.utils import Usage
pytestmark = pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("estimate, saved, expected", [
(None, 0.0, 0.0),
(None, -0.5, -0.5),
(None, 0.5, 0.5),
(None, None, None),
(None, True, None),
(None, float("nan"), None),
({"version": 3, "status": "estimated"}, 0.0, 0.0),
({"version": 2, "status": "estimated"}, -0.5, -0.5),
({"version": 1, "status": "unknown"}, 0.5, None),
({"version": 0, "status": "estimated"}, 0.5, None),
])
def test_known_autorouter_savings_preserves_zero_and_legacy_provenance(
estimate: dict[str, object] | None, saved: object, expected: float | None,
) -> None:
from litellm.proxy.spend_tracking.savings import known_autorouter_savings
result: Final = known_autorouter_savings(
model=None, custom_llm_provider=None, routing_decision=None, usage_object=None,
recorded_autorouter_savings=saved, recorded_autorouter_savings_estimate=estimate,
)
assert result == expected
@pytest.mark.parametrize("model,usage", [
(None, {"cache_read_input_tokens": 100}),
("claude-sonnet-5", None),

View file

@ -2266,6 +2266,7 @@ class TestRunServerDbSetup:
use_migrate=True, use_v2_resolver=True
)
@pytest.mark.parametrize("coverage_failure", [False, True])
@patch("subprocess.run")
@patch("atexit.register")
@patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database")
@ -2278,6 +2279,7 @@ class TestRunServerDbSetup:
mock_setup_database,
mock_atexit_register,
mock_subprocess_run,
coverage_failure: bool,
):
"""Test that proxy exits with code 1 when PrismaManager.setup_database returns False and --enforce_prisma_migration_check is set"""
from litellm.proxy.proxy_cli import run_server
@ -2285,6 +2287,9 @@ class TestRunServerDbSetup:
mock_subprocess_run.return_value = MagicMock(returncode=0)
mock_should_update_schema.return_value = True
mock_setup_database.return_value = False
mock_setup_database.side_effect = (
RuntimeError("daily auto-router coverage installation failed") if coverage_failure else None
)
mock_proxy_module = MagicMock(
app=MagicMock(),
@ -2324,13 +2329,13 @@ class TestRunServerDbSetup:
[
"--local",
"--skip_server_startup",
"--enforce_prisma_migration_check",
"--use_prisma_db_push" if coverage_failure else "--enforce_prisma_migration_check",
],
standalone_mode=False,
)
assert exc_info.value.code == 1
assert exc_info.value.code == (2 if coverage_failure else 1)
mock_setup_database.assert_called_once_with(
use_migrate=True, use_v2_resolver=True
use_migrate=not coverage_failure, use_v2_resolver=True
)
@patch("subprocess.run")

View file

@ -1,5 +1,6 @@
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { fireEvent, render, screen, within } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import React from "react";
import { beforeEach, describe, expect, it, vi } from "vitest";
@ -68,6 +69,9 @@ const totals = (overrides: Partial<Totals> = {}): Totals => ({
avg_session_seconds: 7560,
avg_tokens_per_session: 5_300_000,
spend: 359.86,
llm_spend: (overrides.spend ?? 359.86) - (overrides.classifier_cost ?? 6.146),
cost_coverage: "complete",
cost_requests: overrides.turns ?? 3073,
savings_estimated_turns: overrides.turns ?? 3073,
savings_estimated_actual_spend: overrides.spend ?? 359.86,
classifier_cost: 6.146,
@ -102,6 +106,9 @@ const zeroTotals: Totals = {
avg_session_seconds: 0,
avg_tokens_per_session: 0,
spend: 0,
llm_spend: 0,
cost_coverage: "complete",
cost_requests: 0,
savings_estimated_turns: 0,
savings_estimated_actual_spend: 0,
classifier_cost: 0,
@ -116,6 +123,7 @@ const group = (overrides: Partial<AutoRouterBenchmarkGroup> = {}): AutoRouterBen
router_name: "claude-auto",
router_type: "complexity",
...totals(),
cost_requests: null,
...overrides,
});
@ -157,37 +165,28 @@ describe("AutoRouterBenchmarksTab", () => {
mockAutoRouters();
});
it.each([
{ estimatedTurns: 0, saved: null, pct: null },
{ estimatedTurns: 10, saved: -0.5, pct: -33.3 },
{ estimatedTurns: 10, saved: 0, pct: 0 },
])("preserves costs for $estimatedTurns estimated turns with savings $saved", ({ estimatedTurns, saved, pct }) => {
const cohort = {
it.each([0, 2])("retains historical savings when only %i turns have current estimates", (estimatedTurns) => {
const history: Partial<Totals> = {
savings_estimated_turns: estimatedTurns,
savings_estimated_actual_spend: estimatedTurns ? 2 : 0,
saved_spend: saved,
baseline_spend: estimatedTurns ? 2 + (saved ?? 0) : null,
saved_pct: pct,
saved_per_session: null,
savings_estimated_actual_spend: estimatedTurns ? 0.0004 : 0,
cost_coverage: "partial",
cost_requests: null,
baseline_spend: null,
saved_pct: null,
};
const partial = totals(cohort);
const partial = totals(history);
mockHook({ data: response([], partial) });
renderTab();
expect(screen.getByText("Estimated savings on covered turns")).toBeInTheDocument();
expect(screen.getByText(`${estimatedTurns} of 3,073 turns estimated`)).toBeInTheDocument();
expect(screen.getByText("Total estimated savings")).toBeInTheDocument();
expect(screen.getByText("$2,174.59")).toBeInTheDocument();
expect(screen.queryByText("-86%")).not.toBeInTheDocument();
expect(screen.getByText("$359.86")).toBeInTheDocument();
expect(screen.getByText("Actual spend on covered turns")).toBeInTheDocument();
expect(screen.getByText("Estimated baseline spend on covered turns")).toBeInTheDocument();
expect(screen.getAllByText("Unavailable")).toHaveLength(estimatedTurns ? 1 : 3);
if (saved === 0) {
expect(screen.getByText("0%")).toBeInTheDocument();
expect(screen.getAllByText("$2.00")).toHaveLength(2);
} else if (estimatedTurns) {
expect(screen.getByText("-$0.5000")).toBeInTheDocument();
expect(screen.getByText("+33%")).toBeInTheDocument();
} else {
expect(screen.queryByText("+0%")).not.toBeInTheDocument();
}
expect(screen.getByText("Unavailable")).toBeInTheDocument();
expect(screen.getByText("$23.13")).toBeInTheDocument();
expect(screen.getByText("Estimated baseline spend")).toBeInTheDocument();
expect(screen.queryByText(/turns estimated/)).not.toBeInTheDocument();
expect(screen.queryByText(/older estimates.*excluded|excluded.*older estimates/)).not.toBeInTheDocument();
expect(screen.queryByText("Actual spend on covered turns")).not.toBeInTheDocument();
});
it("leads with total estimated savings, before the four session-shape metrics", () => {
@ -216,13 +215,155 @@ describe("AutoRouterBenchmarksTab", () => {
expect(screen.getByText("-86%")).toBeInTheDocument();
expect(screen.getByText("Actual auto-router spend")).toBeInTheDocument();
expect(screen.getByText("$359.86")).toBeInTheDocument();
expect(screen.getByText("Estimated spend at highest-tier model")).toBeInTheDocument();
expect(screen.getByText("Estimated baseline spend")).toBeInTheDocument();
expect(screen.getByText("$2,534.45")).toBeInTheDocument();
expect(screen.getByText("32.7")).toBeInTheDocument();
expect(screen.getByText("2.1h")).toBeInTheDocument();
expect(screen.getByText("5.3M")).toBeInTheDocument();
});
it("keeps daily savings visible when matching historical costs are unavailable", () => {
const dailyValues: Partial<Totals> = {
spend: null,
llm_spend: null,
cost_coverage: "unavailable",
cost_requests: null,
classifier_cost: null,
baseline_spend: null,
saved_pct: null,
};
const daily = totals(dailyValues);
mockHook({ data: response([group()], daily) });
renderTab();
expect(screen.getByText("$2,174.59")).toBeInTheDocument();
expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([
"Unavailable",
"Unavailable",
"Unavailable",
"Unavailable",
]);
expect(screen.getAllByText("Unavailable")).toHaveLength(4);
expect(screen.getByText("$23.13")).toBeInTheDocument();
expect(screen.queryByText("-86%")).not.toBeInTheDocument();
expect(screen.queryByText(/\/ 1K turns/)).not.toBeInTheDocument();
expect(screen.getByText("Whole sessions overlapping the selected dates")).toBeInTheDocument();
});
it("shows recorded historical costs without presenting partial costs as the full savings comparison", () => {
const recordedCosts: Partial<Totals> = {
spend: 12,
llm_spend: 10,
classifier_cost: null,
cost_coverage: "partial",
cost_requests: null,
baseline_spend: null,
saved_pct: null,
};
mockHook({ data: response([], totals(recordedCosts)) });
renderTab();
expect(screen.getByText("$2,174.59")).toBeInTheDocument();
expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([
"$12.00",
"$10.00",
"Unavailable",
"Unavailable",
]);
expect(screen.getByText("Partial")).toBeInTheDocument();
expect(screen.getByText("Some request costs are unavailable")).toBeInTheDocument();
expect(screen.queryByText("-86%")).not.toBeInTheDocument();
expect(screen.queryByText(/\/ 1K turns/)).not.toBeInTheDocument();
});
it("separates actual spend from the estimated-request comparison and uses request-date classification counts", () => {
const requestCosts: Partial<Totals> = {
spend: 12,
llm_spend: 10,
classifier_cost: 2,
cost_requests: 4,
savings_estimated_turns: 2,
savings_estimated_actual_spend: 4,
saved_spend: 6,
baseline_spend: 10,
saved_pct: 60,
};
mockHook({ data: response([], totals(requestCosts)) });
renderTab();
expect(screen.getByText("$6.00")).toBeInTheDocument();
expect(screen.getByText("-60%")).toBeInTheDocument();
expect(screen.getByText("2 of 4 requests have savings estimates")).toBeInTheDocument();
expect(screen.getByText("Actual spend on estimated requests")).toBeInTheDocument();
expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([
"$12.00",
"$10.00",
"$2.00",
"$4.00",
"$10.00",
]);
expect(screen.getByText("($500.00 / 1K turns)")).toBeInTheDocument();
});
it("labels a selected router as session usage and keeps its recorded cost breakdown", async () => {
const user = userEvent.setup();
const dailyValues: Partial<Totals> = {
saved_spend: 3_000,
spend: null,
llm_spend: null,
cost_coverage: "unavailable",
cost_requests: null,
classifier_cost: null,
baseline_spend: null,
saved_pct: null,
};
const daily = totals(dailyValues);
mockHook({ data: response([group()], daily) });
renderTab();
expect(screen.getByText("$3,000.00")).toBeInTheDocument();
await user.click(screen.getByRole("combobox"));
await user.click(await screen.findByRole("option", { name: "claude-auto" }));
expect(screen.getByRole("heading", { name: "Auto-router session usage" })).toBeInTheDocument();
expect(screen.getByText("Whole sessions overlapping the selected dates")).toBeInTheDocument();
expect(screen.getByText("$2,174.59")).toBeInTheDocument();
expect(screen.queryByText("$3,000.00")).not.toBeInTheDocument();
expect(screen.getByText("-86%")).toBeInTheDocument();
expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([
"$359.86",
"$353.71",
"$6.15",
"$2,534.45",
]);
});
it("keeps selected-router spend visible without a baseline or percentage for unestimated turns", async () => {
const user = userEvent.setup();
const unknownSavings: Partial<AutoRouterBenchmarkGroup> = {
savings_estimated_turns: 0,
savings_estimated_actual_spend: 0,
saved_spend: 0,
baseline_spend: null,
saved_pct: null,
saved_per_session: 0,
};
mockHook({ data: response([group(unknownSavings)]) });
renderTab();
await user.click(screen.getByRole("combobox"));
await user.click(await screen.findByRole("option", { name: "claude-auto" }));
expect(screen.getByRole("heading", { name: "Auto-router session usage" })).toBeInTheDocument();
expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([
"$359.86",
"$353.71",
"$6.15",
"Unavailable",
]);
expect(screen.queryByText("0%")).not.toBeInTheDocument();
});
it.each([
{ spend: 20665.28, classifier_cost: 342.18, turns: 140815, llm: "$20,323.10", cost: "$342.18", rate: "$2.43" },
{ spend: 0, classifier_cost: 0, turns: 0, llm: "$0.00", cost: "$0.00", rate: "$0.00" },
@ -243,15 +384,31 @@ describe("AutoRouterBenchmarksTab", () => {
});
it.each([null, undefined])("keeps totals when the classification breakdown is %s", (classifier_cost) => {
const stats = totals({ classifier_cost });
const partialCosts: Partial<Totals> = {
classifier_cost,
cost_coverage: "partial",
baseline_spend: null,
saved_pct: null,
};
const stats = totals(partialCosts);
mockHook({ data: response([group(stats)], stats) });
renderTab();
expect(screen.getAllByText("Unavailable")).toHaveLength(2);
expect(screen.queryByText(/\/ 1K turns/)).not.toBeInTheDocument();
expect(screen.getByText("$359.86")).toBeInTheDocument();
expect(screen.getByText("$353.71")).toBeInTheDocument();
expect(screen.getByText("$2,174.59")).toBeInTheDocument();
expect(screen.getByText(/some usage predates classification-cost tracking/)).toBeInTheDocument();
});
it.each([
{ llm_spend: undefined, displayed: "$353.71" },
{ llm_spend: null, displayed: "Unavailable" },
])("only derives LLM spend when an older proxy omits its subtotal: %j", ({ llm_spend, displayed }) => {
mockHook({ data: response([], totals({ llm_spend })) });
renderTab();
expect(screen.getAllByRole("definition")[1]).toHaveTextContent(displayed);
});
it("pairs the savings with the session count it was earned over, in its own tile", () => {
@ -275,7 +432,7 @@ describe("AutoRouterBenchmarksTab", () => {
"Actual auto-router spend",
"LLM spend",
"Classification cost($2.00 / 1K turns)",
"Estimated spend at highest-tier model",
"Estimated baseline spend",
]);
expect(values).toEqual(["$359.86", "$353.71", "$6.15", "$2,534.45"]);
});
@ -441,7 +598,11 @@ describe("AutoRouterBenchmarksTab", () => {
expect(onDateChange).toHaveBeenCalledWith({ from: new Date(2026, 7, 1), to: new Date(2026, 7, 5) });
});
it("scopes the query to one key when the usage view is mounted for a key", () => {
it.each([
{ apiKey: "key-hash-1", userId: undefined },
{ apiKey: undefined, userId: "user-1" },
{ apiKey: "key-hash-1", userId: "user-1" },
])("preserves the key/user filters for the usage view: %j", ({ apiKey, userId }) => {
mockHook({ data: response([group()]) });
const dateValue = { from: new Date(2026, 6, 6), to: new Date(2026, 7, 5) };
const activity = {
@ -456,11 +617,11 @@ describe("AutoRouterBenchmarksTab", () => {
};
render(
<QueryClientProvider client={new QueryClient()}>
<AutoRouterUsageView accessToken="sk-test" activity={activity} apiKey="key-hash-1" />
<AutoRouterUsageView accessToken="sk-test" activity={activity} apiKey={apiKey} userId={userId} />
</QueryClientProvider>,
);
expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, "key-hash-1", undefined);
expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, apiKey, userId);
expect(screen.getByText("Total estimated savings")).toBeInTheDocument();
expect(screen.queryByRole("tab", { name: "Shadow Evals" })).not.toBeInTheDocument();
});

View file

@ -25,6 +25,7 @@ import {
groupLabel,
pctLabel,
viewFor,
viewGroup,
type AutoRouterBenchmarksResponse,
type AutoRouterCacheStats,
type BenchmarkView,
@ -71,16 +72,60 @@ const SpendRow: React.FC<{ label: string; value: string; hint?: string; subdued?
</dl>
);
const CostBreakdown: React.FC<{ view: BenchmarkView; partialComparison: boolean }> = ({ view, partialComparison }) => {
const stats = view.stats;
const costTurns = viewGroup(view) ? stats.turns : stats.cost_requests;
const llmSpend =
stats.llm_spend === undefined && stats.spend != null && stats.classifier_cost != null
? stats.spend - stats.classifier_cost
: stats.llm_spend;
return (
<div className="flex flex-col justify-center border-t p-6 md:border-t-0 md:border-l">
<SpendRow
label="Actual auto-router spend"
value={stats.spend == null ? "Unavailable" : usd(stats.spend)}
hint={stats.cost_coverage === "partial" ? "Partial" : undefined}
/>
<div className="mb-3 border-l-2 pl-4">
<SpendRow subdued label="LLM spend" value={llmSpend == null ? "Unavailable" : usd(llmSpend)} />
<SpendRow
subdued
label="Classification cost"
value={stats.classifier_cost == null ? "Unavailable" : usd(stats.classifier_cost)}
hint={
stats.classifier_cost == null || costTurns == null
? undefined
: classificationRatePer1kTurns(stats.classifier_cost, costTurns)
}
/>
</div>
{stats.cost_coverage === "partial" && (
<p className="mb-3 text-xs text-muted-foreground">Some request costs are unavailable</p>
)}
<Separator />
{partialComparison && stats.baseline_spend != null && (
<SpendRow label="Actual spend on estimated requests" value={usd(stats.savings_estimated_actual_spend)} />
)}
<SpendRow
label="Estimated baseline spend"
value={stats.baseline_spend == null ? "Unavailable" : usd(stats.baseline_spend)}
/>
</div>
);
};
const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => {
const stats = view.stats;
const cheaper = stats.saved_spend != null && stats.saved_spend >= 0;
const completeCoverage = stats.savings_estimated_turns === stats.turns;
const dailyCosts = viewGroup(view) === null;
const partialComparison =
dailyCosts && stats.cost_requests != null && stats.savings_estimated_turns < stats.cost_requests;
return (
<Card className="overflow-hidden py-0">
<div className="grid md:grid-cols-[minmax(0,1fr)_minmax(0,1fr)]">
<div className="flex flex-col items-center justify-center gap-2 p-6">
<p className="text-xs font-semibold uppercase tracking-wider text-muted-foreground">
{completeCoverage ? "Total estimated savings" : "Estimated savings on covered turns"}
Total estimated savings
</p>
<div className="flex flex-wrap items-center justify-center gap-3">
<p className="min-w-0 break-all text-center text-4xl font-semibold tracking-tight text-foreground xl:text-6xl">
@ -96,51 +141,15 @@ const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => {
</Badge>
)}
</div>
<p className="text-center text-xs text-muted-foreground">
{stats.savings_estimated_turns.toLocaleString()} of {stats.turns.toLocaleString()} turns estimated
</p>
{!completeCoverage && (
{partialComparison && (
<p className="text-center text-xs text-muted-foreground">
Turns without a current estimate are excluded, including older estimates.
{stats.savings_estimated_turns.toLocaleString()} of {stats.cost_requests?.toLocaleString()} requests have
savings estimates
</p>
)}
</div>
<div className="flex flex-col justify-center border-t p-6 md:border-t-0 md:border-l">
<SpendRow label="Actual auto-router spend" value={usd(stats.spend)} />
<div className="mb-3 border-l-2 pl-4">
<SpendRow
subdued
label="LLM spend"
value={stats.classifier_cost == null ? "Unavailable" : usd(stats.spend - stats.classifier_cost)}
/>
<SpendRow
subdued
label="Classification cost"
value={stats.classifier_cost == null ? "Unavailable" : usd(stats.classifier_cost)}
hint={
stats.classifier_cost == null
? undefined
: classificationRatePer1kTurns(stats.classifier_cost, stats.turns)
}
/>
</div>
{stats.classifier_cost == null && (
<p className="mb-3 text-xs text-muted-foreground">
Breakdown unavailable because some usage predates classification-cost tracking.
</p>
)}
<Separator />
{!completeCoverage && (
<SpendRow label="Actual spend on covered turns" value={usd(stats.savings_estimated_actual_spend)} />
)}
<SpendRow
label={
completeCoverage ? "Estimated spend at highest-tier model" : "Estimated baseline spend on covered turns"
}
value={stats.baseline_spend == null ? "Unavailable" : usd(stats.baseline_spend)}
/>
</div>
<CostBreakdown view={view} partialComparison={partialComparison} />
</div>
</Card>
);
@ -293,6 +302,9 @@ const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data,
<>
<HeroCard view={view} />
{!viewGroup(view) && (
<p className="text-xs text-muted-foreground">Whole sessions overlapping the selected dates</p>
)}
<TierTurnsChart view={view} autoRouters={autoRouters} />
<div className="grid grid-cols-1 gap-4 sm:grid-cols-2 lg:grid-cols-4">
@ -306,15 +318,6 @@ const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data,
<Metric label="Avg tokens per session" value={formatNumberWithCommas(stats.avg_tokens_per_session, 1, true)} />
</div>
<p className="text-xs text-muted-foreground">
Compares covered turns with the estimated cost of using the router&apos;s highest-tier baseline model. Estimates
use registered requests since tracking began, matching cache prefixes and expiry, and the actual response
length. Total actual spend includes every turn; savings and baseline spend include only turns with a current
estimate, including turns with zero savings. Savings are net of recorded LLM classification cost. Classification
cost per 1K turns is averaged over all auto-router turns, including those that skip classification. The range
counts whole sessions that overlap it, so totals can differ from savings views that group usage by UTC day.
</p>
<div className="space-y-4">
<div className="flex flex-wrap items-baseline gap-2">
<h3 className="text-lg font-semibold text-foreground">Auto-router prompt caching</h3>
@ -347,15 +350,22 @@ export const AutoRouterUsageView: React.FC<AutoRouterBenchmarksTabProps> = ({
const { data: autoRouters } = useAutoRouters();
const groups = data?.groups ?? [];
const selectedLabel = data ? viewFor(data, selectedKey).label : "All auto-routers";
const selectedView = data ? viewFor(data, selectedKey) : null;
const selectedLabel = selectedView?.label ?? "All auto-routers";
const selectedGroup = selectedView ? viewGroup(selectedView) : null;
const rangeLabel = formatRangeLabel(dateValue.from, dateValue.to);
return (
<div className="w-full space-y-6">
<div className="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between">
<div>
<h2 className="text-xl font-semibold text-foreground">Auto-router usage</h2>
<h2 className="text-xl font-semibold text-foreground">
{selectedGroup ? "Auto-router session usage" : "Auto-router usage"}
</h2>
{rangeLabel && <p className="mt-1 text-sm text-muted-foreground">{rangeLabel} (UTC)</p>}
{selectedGroup && (
<p className="mt-1 text-xs text-muted-foreground">Whole sessions overlapping the selected dates</p>
)}
</div>
<div className="flex w-full flex-col gap-3 sm:w-auto sm:flex-row sm:items-center">
<AdvancedDatePicker value={dateValue} onValueChange={onDateChange} />
@ -379,8 +389,7 @@ export const AutoRouterUsageView: React.FC<AutoRouterBenchmarksTabProps> = ({
{userId && (
<p className="text-sm text-muted-foreground">
Usage for this user across API keys and JWT-authenticated requests. Older sessions recorded without a user ID
are not included.
Usage for this user across API keys and JWT-authenticated requests.
</p>
)}
<BenchmarksBody

View file

@ -12,15 +12,19 @@ vi.mock("@/components/shared/charts", () => ({
}));
import TierTurnsChart, { tierDisplayLabel } from "./TierTurnsChart";
import type { AutoRouterBenchmarkGroup, BenchmarkView } from "./autoRouterBenchmarks";
import type { AutoRouterBenchmarkGroup, AutoRouterBenchmarkTotals, BenchmarkView } from "./autoRouterBenchmarks";
const totalsOnly = {
const totalsOnly: AutoRouterBenchmarkTotals = {
sessions: 3,
turns: 9,
avg_turns_per_session: 3,
avg_session_seconds: 60,
avg_tokens_per_session: 100,
spend: 1,
classifier_cost: 0,
llm_spend: 1,
cost_coverage: "complete",
cost_requests: null,
savings_estimated_turns: 9,
savings_estimated_actual_spend: 1,
saved_spend: 1,

View file

@ -37,6 +37,9 @@ const totals = (overrides: Partial<AutoRouterBenchmarkGroup> = {}) => ({
avg_session_seconds: 7560,
avg_tokens_per_session: 5_300_000,
spend: 359.86,
llm_spend: 353.714,
cost_coverage: "complete" as const,
cost_requests: null,
savings_estimated_turns: overrides.turns ?? 3073,
savings_estimated_actual_spend: overrides.spend ?? 359.86,
classifier_cost: 6.146,

View file

@ -342,14 +342,17 @@ const routerUsageResponse = (saved: number): AutoRouterBenchmarksResponse => ({
avg_turns_per_session: 1,
avg_session_seconds: 0,
avg_tokens_per_session: 100,
spend: 10,
spend: null,
llm_spend: null,
cost_coverage: "unavailable",
cost_requests: null,
savings_estimated_turns: 2,
savings_estimated_actual_spend: 10,
classifier_cost: 0,
classifier_cost: null,
saved_spend: saved,
baseline_spend: 10 + saved,
saved_pct: (100 * saved) / (10 + saved),
saved_per_session: saved / 2,
baseline_spend: null,
saved_pct: null,
saved_per_session: 3,
cache: {
coverage_pct: 100,
hit_rate_pct: 0,
@ -405,7 +408,17 @@ describe("UserInfoView auto-router usage", () => {
const params = new URL(request.url).searchParams;
expect(params.get("user_id")).toBe("user-123");
expect(params.has("api_key")).toBe(false);
expect(screen.getByText(/Older sessions recorded without a user ID are not included/)).toBeInTheDocument();
expect(
screen.getByText("Usage for this user across API keys and JWT-authenticated requests."),
).toBeInTheDocument();
expect(screen.getByText("$3.00")).toBeInTheDocument();
expect(screen.getByText("Estimated baseline spend")).toBeInTheDocument();
expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([
"Unavailable",
"Unavailable",
"Unavailable",
"Unavailable",
]);
},
);

View file

@ -1,6 +1,8 @@
import { screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { AutoRouterBenchmarksResponse } from "@/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks";
import { renderWithProviders, testQueryClient } from "../../../tests/test-utils";
import KeyAutoRouterUsageTab from "./KeyAutoRouterUsageTab";
@ -32,6 +34,9 @@ const stats = {
avg_session_seconds: 30,
avg_tokens_per_session: 100,
spend: 1.25,
llm_spend: 1,
cost_coverage: "complete" as const,
cost_requests: null,
savings_estimated_turns: 4,
savings_estimated_actual_spend: 1.25,
classifier_cost: 0.25,
@ -42,11 +47,20 @@ const stats = {
cache,
};
const benchmarks = {
const benchmarks: AutoRouterBenchmarksResponse = {
start_date: "2025-01-01",
end_date: "2025-01-31",
routers_in_scope: 2,
totals: stats,
totals: {
...stats,
saved_spend: 12.75,
spend: null,
llm_spend: null,
cost_coverage: "unavailable",
classifier_cost: null,
baseline_spend: null,
saved_pct: null,
},
groups: [
{ router_name: "router-one", router_type: "complexity", tier_turns: { SIMPLE: 4 }, ...stats },
{
@ -55,6 +69,7 @@ const benchmarks = {
tier_turns: { SIMPLE: 1 },
...stats,
spend: 0.25,
llm_spend: 0,
saved_spend: 0.75,
baseline_spend: 1,
},
@ -77,14 +92,38 @@ describe("KeyAutoRouterUsageTab", () => {
vi.stubGlobal("fetch", fetchMock);
});
it("renders this key's spend, baseline, savings and per-router filter", async () => {
it("renders this key's daily savings and selected-router session costs", async () => {
const user = userEvent.setup();
const activity = {
dateValue: { from: new Date(2025, 0, 1), to: new Date(2025, 0, 31) },
onDateChange: vi.fn(),
};
renderWithProviders(<KeyAutoRouterUsageTab accessToken="test-token" keyToken="key-hash-1" activity={activity} />);
expect(await screen.findByText("$8.75")).toBeInTheDocument();
expect(await screen.findByText("$12.75")).toBeInTheDocument();
expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([
"Unavailable",
"Unavailable",
"Unavailable",
"Unavailable",
]);
expect(screen.getByText("$4.38")).toBeInTheDocument();
expect(screen.queryByText("-88%")).not.toBeInTheDocument();
expect(screen.getByText("All auto-routers")).toBeInTheDocument();
const benchmarkUrl = new URL(requestedUrls().find((url) => url.includes("/auto_router/benchmarks")) ?? "");
expect(benchmarkUrl.searchParams.get("api_key")).toBe("key-hash-1");
expect(benchmarkUrl.searchParams.get("start_date")).toBe("2025-01-01");
expect(benchmarkUrl.searchParams.get("end_date")).toBe("2025-01-31");
await user.click(screen.getByRole("combobox"));
await user.click(await screen.findByRole("option", { name: "router-one" }));
expect(screen.getByRole("heading", { name: "Auto-router session usage" })).toBeInTheDocument();
expect(screen.getByText("Whole sessions overlapping the selected dates")).toBeInTheDocument();
expect(screen.getByText("$8.75")).toBeInTheDocument();
expect(screen.queryByText("$12.75")).not.toBeInTheDocument();
expect(screen.getByText("-88%")).toBeInTheDocument();
expect(screen.getByText("Actual auto-router spend")).toBeInTheDocument();
expect(screen.getByText("$1.25")).toBeInTheDocument();
expect(screen.getByText("LLM spend")).toBeInTheDocument();
@ -92,15 +131,9 @@ describe("KeyAutoRouterUsageTab", () => {
expect(screen.getByText("Classification cost")).toBeInTheDocument();
expect(screen.getByText("$0.2500")).toBeInTheDocument();
expect(screen.getByText("($62.50 / 1K turns)")).toBeInTheDocument();
expect(screen.getByText("Estimated spend at highest-tier model")).toBeInTheDocument();
expect(screen.getByText("Estimated baseline spend")).toBeInTheDocument();
expect(screen.getByText("$10.00")).toBeInTheDocument();
expect(screen.getByText("Auto-router prompt caching")).toBeInTheDocument();
expect(screen.getAllByText("50.0%").length).toBeGreaterThan(0);
expect(screen.getByText("All auto-routers")).toBeInTheDocument();
const benchmarkUrl = new URL(requestedUrls().find((url) => url.includes("/auto_router/benchmarks")) ?? "");
expect(benchmarkUrl.searchParams.get("api_key")).toBe("key-hash-1");
expect(benchmarkUrl.searchParams.get("start_date")).toBe("2025-01-01");
expect(benchmarkUrl.searchParams.get("end_date")).toBe("2025-01-31");
});
});

View file

@ -1263,12 +1263,12 @@ export interface paths {
* @description Benchmarks for the auto-router dashboard: session shape, savings against the configured
* baseline, and prompt-caching behaviour bucketed by what the router did.
*
* Reads session rollups folded once per request at spend-write time, so this endpoint
* never scans LiteLLM_SpendLogs. A user filter selects only turns attributed to that
* internal user when written; older key-only history remains outside user views. A session
* is in the window when it overlaps it: its last turn is on or after start_date and its first turn is on or before
* end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is
* over that bucket's turns.
* Total savings use the same request-date daily aggregation as Overall cost optimization,
* including recorded history and requests without session IDs. Costs use matching request
* dates with explicit coverage for history predating daily router cost tracking. Session
* statistics and per-router groups cover whole sessions that overlap the window.
* A user filter uses the user recorded when each row was written.
* Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is over its turns.
*
* The rollup supplies the measures, never the list. Which routers appear comes from the
* model registry, so one shows up as soon as it is configured and reads zero until it
@ -24180,7 +24180,7 @@ export interface components {
};
/**
* AutoRouterBenchmarkGroup
* @description One auto-router's slice of the benchmarks.
* @description One auto-router's whole overlapping sessions, separate from request-date daily totals.
*/
AutoRouterBenchmarkGroup: {
/** Avg Session Seconds */
@ -24191,15 +24191,32 @@ export interface components {
avg_turns_per_session: number;
/**
* Baseline Spend
* @description Estimated single-model cost for covered turns only
* @description Matching actual spend plus recorded savings. Daily totals compare estimated requests only; null when historical cost coverage or the savings estimate is unavailable
*/
baseline_spend: number | null;
cache: components["schemas"]["AutoRouterCacheStats"];
/**
* Classifier Cost
* @description Recorded LLM classifier cost already included in spend; null when any session turns predate subtotal recording, and zero for an empty window
* @description Classifier charges included in actual cost; null when any contributing request lacks the subtotal
*/
classifier_cost: number | null;
/**
* Cost Coverage
* @description Whether recorded costs cover the entire selected population
* @default unavailable
* @enum {string}
*/
cost_coverage: "complete" | "partial" | "unavailable";
/**
* Cost Requests
* @description Number of requests in the cost population; null when historical coverage is incomplete
*/
cost_requests?: number | null;
/**
* Llm Spend
* @description Routed LLM cost excluding classifier charges
*/
llm_spend?: number | null;
/**
* Router Name
* @description The auto-router alias requests were sent to
@ -24212,36 +24229,36 @@ export interface components {
router_type: string;
/**
* Saved Pct
* @description Covered savings over covered baseline spend, as a percentage
* @description Recorded savings divided by the matching estimated baseline; null when that baseline is unavailable
*/
saved_pct: number | null;
/**
* Saved Per Session
* @description Average session savings; unavailable unless every turn is covered
* @description Recorded savings in whole overlapping sessions divided by their session count, independent of request-date daily savings totals
*/
saved_per_session: number | null;
/**
* Saved Spend
* @description Signed savings for covered turns only; null when traffic has no current estimates
* @description Signed recorded savings, including historical and current estimates once. Totals use the same request-date daily source as Overall cost optimization, including requests without sessions. Per-router groups cover whole overlapping sessions. Turns without an estimate add no savings
*/
saved_spend: number | null;
/**
* Savings Estimated Actual Spend
* @description Actual spend, including classifier cost, for covered turns only
* @description Actual spend including classification for requests covered by the savings comparison
*/
savings_estimated_actual_spend: number;
/**
* Savings Estimated Turns
* @description Turns covered by the current savings estimator; legacy estimates are excluded
* @description Requests with an available savings comparison. Totals use request dates and groups use session coverage. Historical savings remain included even when their coverage was not tracked
*/
savings_estimated_turns: number;
/** Sessions */
sessions: number;
/**
* Spend
* @description What the routed traffic actually cost
* @description Actual auto-router cost including classification. Totals use request dates; groups cover whole overlapping sessions. Partial recorded history is identified by cost_coverage
*/
spend: number;
spend: number | null;
/**
* Tier Turns
* @description Turns per tier, keyed by the tier name the routing decision recorded at request time (never re-derived at read time, since the tier-to-model mapping is mutable config). Tier names are scoped to this group's router_type and are not comparable across types: a complexity router reports 'SIMPLE'/'MEDIUM'/'COMPLEX'/'REASONING', a quality router reports its numeric quality tier, and an adaptive router records no tier at all. Turns no tier served (the classifier fell back to default_model) are absent rather than pooled under a sentinel key, so the values may sum to less than turns
@ -24254,7 +24271,7 @@ export interface components {
};
/**
* AutoRouterBenchmarkTotals
* @description Session-shape and savings aggregates over auto-routed traffic in the window.
* @description Request-date savings totals with statistics for whole sessions overlapping the window.
*/
AutoRouterBenchmarkTotals: {
/** Avg Session Seconds */
@ -24265,53 +24282,70 @@ export interface components {
avg_turns_per_session: number;
/**
* Baseline Spend
* @description Estimated single-model cost for covered turns only
* @description Matching actual spend plus recorded savings. Daily totals compare estimated requests only; null when historical cost coverage or the savings estimate is unavailable
*/
baseline_spend: number | null;
cache: components["schemas"]["AutoRouterCacheStats"];
/**
* Classifier Cost
* @description Recorded LLM classifier cost already included in spend; null when any session turns predate subtotal recording, and zero for an empty window
* @description Classifier charges included in actual cost; null when any contributing request lacks the subtotal
*/
classifier_cost: number | null;
/**
* Cost Coverage
* @description Whether recorded costs cover the entire selected population
* @default unavailable
* @enum {string}
*/
cost_coverage: "complete" | "partial" | "unavailable";
/**
* Cost Requests
* @description Number of requests in the cost population; null when historical coverage is incomplete
*/
cost_requests?: number | null;
/**
* Llm Spend
* @description Routed LLM cost excluding classifier charges
*/
llm_spend?: number | null;
/**
* Saved Pct
* @description Covered savings over covered baseline spend, as a percentage
* @description Recorded savings divided by the matching estimated baseline; null when that baseline is unavailable
*/
saved_pct: number | null;
/**
* Saved Per Session
* @description Average session savings; unavailable unless every turn is covered
* @description Recorded savings in whole overlapping sessions divided by their session count, independent of request-date daily savings totals
*/
saved_per_session: number | null;
/**
* Saved Spend
* @description Signed savings for covered turns only; null when traffic has no current estimates
* @description Signed recorded savings, including historical and current estimates once. Totals use the same request-date daily source as Overall cost optimization, including requests without sessions. Per-router groups cover whole overlapping sessions. Turns without an estimate add no savings
*/
saved_spend: number | null;
/**
* Savings Estimated Actual Spend
* @description Actual spend, including classifier cost, for covered turns only
* @description Actual spend including classification for requests covered by the savings comparison
*/
savings_estimated_actual_spend: number;
/**
* Savings Estimated Turns
* @description Turns covered by the current savings estimator; legacy estimates are excluded
* @description Requests with an available savings comparison. Totals use request dates and groups use session coverage. Historical savings remain included even when their coverage was not tracked
*/
savings_estimated_turns: number;
/** Sessions */
sessions: number;
/**
* Spend
* @description What the routed traffic actually cost
* @description Actual auto-router cost including classification. Totals use request dates; groups cover whole overlapping sessions. Partial recorded history is identified by cost_coverage
*/
spend: number;
spend: number | null;
/** Turns */
turns: number;
};
/**
* AutoRouterBenchmarksResponse
* @description Benchmarks for the auto-router dashboard, aggregated from the per-session rollup.
* @description Daily recorded savings with tracked-session breakdowns for the auto-router dashboard.
*/
AutoRouterBenchmarksResponse: {
/**
@ -24573,19 +24607,19 @@ export interface components {
AutoRouterSessionResponse: {
/**
* Baseline Model
* @description The savings baseline most covered turns were priced against, recorded turn by turn, so it still names the counterfactual after the router is reconfigured or removed. None when no turn recorded one: rows from before the baseline was recorded, and adaptive and quality routers, which derive no baseline and so report no savings
* @description The savings baseline recorded by most session turns, including historical turns, recorded turn by turn, so it still names the counterfactual after the router is reconfigured or removed. None when no turn recorded one: rows from before the baseline was recorded, and adaptive and quality routers, which derive no baseline and so report no savings
*/
baseline_model: string | null;
/**
* Baseline Models
* @description Covered turns priced against each baseline model; more than one entry means the router's baseline changed mid-session and baseline_spend mixes both
* @description Session turns recording each baseline model, including historical turns; more than one entry means the router's baseline changed mid-session. These counts do not imply savings-estimate coverage
*/
baseline_models: {
[key: string]: number;
};
/**
* Baseline Spend
* @description Estimated single-model cost; unavailable unless every turn is covered
* @description Actual session spend plus recorded savings; turns without an estimate add no savings
*/
baseline_spend: number | null;
/**
@ -24605,7 +24639,7 @@ export interface components {
router_type: string;
/**
* Saved Spend
* @description Estimated savings for covered turns only, net of classifier cost
* @description Recorded signed savings, retaining historical estimates and adding current estimates once
*/
saved_spend: number | null;
/**