From b0b24e7e5ca4f5e6b106665bf8bab18c15435b21 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 22 Sep 2026 18:56:17 -0700 Subject: [PATCH] fix(autorouter): preserve historical savings and align dashboard totals --- .../migration.sql | 172 +++++++++ .../litellm_proxy_extras/schema.prisma | 8 + .../litellm_proxy_extras/utils.py | 33 ++ litellm/models/autorouter_session.py | 11 +- litellm/proxy/_types.py | 7 + .../client/cli/commands/statusline_script.py | 43 ++- litellm/proxy/db/autorouter_daily_spend.py | 75 ++++ .../proxy/db/autorouter_historical_spend.py | 181 ++++++++++ litellm/proxy/db/daily_spend_bulk_upsert.py | 43 ++- litellm/proxy/db/db_spend_update_writer.py | 34 ++ .../daily_spend_update_queue.py | 21 ++ litellm/proxy/db/prisma_client.py | 2 + .../auto_router_endpoints.py | 104 +++++- litellm/proxy/schema.prisma | 8 + litellm/proxy/spend_tracking/savings.py | 72 ++-- .../auto_router_endpoints.py | 55 ++- schema.prisma | 8 + .../test_litellm_proxy_extras_utils.py | 21 +- tests/proxy_behavior/spend/conftest.py | 16 +- .../spend/test_autorouter_daily_spend.py | 273 +++++++++++++++ .../spend/test_baseline_accounting.py | 177 ++++++++++ .../test_autorouter_baseline_state.py | 318 +++++++++++++++++ .../client/cli/test_statusline_script.py | 59 ++++ tests/test_litellm/proxy/db/conftest.py | 11 + .../test_daily_spend_update_queue.py | 35 +- .../db/test_autorouter_historical_spend.py | 286 +++++++++++++++ .../proxy/db/test_daily_spend_bulk_upsert.py | 19 +- .../proxy/db/test_db_spend_update_writer.py | 101 ++++++ .../proxy/db/test_prisma_client.py | 55 ++- .../test_auto_router_endpoints.py | 330 ++++++++++++++++-- .../test_common_daily_activity.py | 7 + .../test_daily_global_spend_rollup.py | 7 + .../proxy/spend_tracking/test_savings.py | 24 ++ tests/test_litellm/proxy/test_proxy_cli.py | 11 +- .../AutoRouterBenchmarksTab.test.tsx | 227 ++++++++++-- .../_components/AutoRouterBenchmarksTab.tsx | 119 ++++--- .../_components/TierTurnsChart.test.tsx | 8 +- .../_components/autoRouterBenchmarks.test.ts | 3 + .../user_info_view.integration.test.tsx | 25 +- ...KeyAutoRouterUsageTab.integration.test.tsx | 55 ++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 96 +++-- 41 files changed, 2872 insertions(+), 288 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260923000000_add_daily_autorouter_costs/migration.sql create mode 100644 litellm/proxy/db/autorouter_daily_spend.py create mode 100644 litellm/proxy/db/autorouter_historical_spend.py create mode 100644 tests/proxy_behavior/spend/test_autorouter_daily_spend.py create mode 100644 tests/test_litellm/proxy/db/test_autorouter_historical_spend.py diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260923000000_add_daily_autorouter_costs/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260923000000_add_daily_autorouter_costs/migration.sql new file mode 100644 index 00000000000..4821f7b309a --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260923000000_add_daily_autorouter_costs/migration.sql @@ -0,0 +1,172 @@ +BEGIN; +SET LOCAL lock_timeout = '10s'; +SET LOCAL statement_timeout = '30s'; + +DO $install$ +DECLARE + target_schema CONSTANT TEXT := current_schema(); + daily_table CONSTANT REGCLASS := to_regclass(format('%I.%I', target_schema, 'LiteLLM_DailyUserSpend')); + observation_table CONSTANT REGCLASS := to_regclass(format('%I.%I', target_schema, 'LiteLLM_AutoRouterBaselineObservation')); + -- This marker versions the full column/function/trigger contract, preserved by older pods. + implementation_version CONSTANT INTEGER := 1; + function_body CONSTANT TEXT := $function$ +DECLARE + previous_publication JSONB := OLD.daily_costs_publication::jsonb; + next_publication JSONB := NEW.publication::jsonb; + attribution JSONB := NEW.data::jsonb -> 'daily'; + previous_estimated BOOLEAN := COALESCE(previous_publication ->> 'status' = 'estimated', FALSE) + AND previous_publication ->> 'actual_spend' IS NOT NULL + AND previous_publication ->> 'baseline_spend' IS NOT NULL; + next_estimated BOOLEAN := COALESCE(next_publication ->> 'status' = 'estimated', FALSE) + AND next_publication ->> 'actual_spend' IS NOT NULL + AND next_publication ->> 'baseline_spend' IS NOT NULL; + request_delta BIGINT := (CASE WHEN next_estimated THEN 1 ELSE 0 END) + - (CASE WHEN previous_estimated THEN 1 ELSE 0 END); + actual_delta DOUBLE PRECISION := + (CASE WHEN next_estimated THEN (next_publication ->> 'actual_spend')::double precision ELSE 0 END) + - (CASE WHEN previous_estimated THEN (previous_publication ->> 'actual_spend')::double precision ELSE 0 END); + target_user_id TEXT; +BEGIN + NEW.daily_costs_publication := NEW.publication; + IF request_delta = 0 AND actual_delta = 0 THEN + RETURN NEW; + END IF; + + FOR target_user_id IN + SELECT DISTINCT COALESCE(target ->> 'entity_id', '') + FROM jsonb_array_elements(COALESCE(attribution -> 'targets', '[]'::jsonb)) AS target + WHERE target ->> 'entity' = 'user' + ORDER BY 1 + LOOP + INSERT INTO "LiteLLM_DailyUserSpend" ( + id, user_id, date, api_key, model, custom_llm_provider, + mcp_namespaced_tool_name, endpoint, model_group, + autorouter_estimated_requests, autorouter_estimated_actual_spend, updated_at + ) VALUES ( + 'autorouter-coverage:' || length(NEW.request_id)::text || ':' || NEW.request_id || ':' || target_user_id, + target_user_id, COALESCE(attribution ->> 'date', ''), COALESCE(attribution ->> 'api_key', ''), + COALESCE(attribution ->> 'model', ''), COALESCE(attribution ->> 'custom_llm_provider', ''), + COALESCE(attribution ->> 'mcp_namespaced_tool_name', ''), COALESCE(attribution ->> 'endpoint', ''), + attribution ->> 'model_group', request_delta, actual_delta, (NOW() AT TIME ZONE 'UTC') + ) + ON CONFLICT (user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint) + DO UPDATE SET + autorouter_estimated_requests = "LiteLLM_DailyUserSpend".autorouter_estimated_requests + + EXCLUDED.autorouter_estimated_requests, + autorouter_estimated_actual_spend = "LiteLLM_DailyUserSpend".autorouter_estimated_actual_spend + + EXCLUDED.autorouter_estimated_actual_spend, + updated_at = (NOW() AT TIME ZONE 'UTC'); + END LOOP; + RETURN NEW; +END; +$function$; + missing_columns BOOLEAN; + invalid_columns BOOLEAN; + function_id OID; + function_version INTEGER; + function_ready BOOLEAN; + trigger_ready BOOLEAN; +BEGIN + IF daily_table IS NULL OR observation_table IS NULL THEN + RAISE EXCEPTION 'Daily auto-router coverage requires its tables in schema %', target_schema; + END IF; + + FOR attempt IN 1..2 LOOP + SELECT bool_or(a.attnum IS NULL), + bool_or(a.attnum IS NOT NULL AND ( + a.atttypid <> required.column_type::regtype + OR a.attnotnull <> required.not_null + OR CASE WHEN required.not_null + THEN COALESCE(pg_get_expr(d.adbin, d.adrelid), '') !~ '^0([.]0+)?(::(bigint|double precision))?$' + ELSE d.oid IS NOT NULL END + )) + INTO missing_columns, invalid_columns + FROM (VALUES + (daily_table, 'autorouter_accounted_requests', 'bigint', TRUE), + (daily_table, 'autorouter_requests', 'bigint', TRUE), + (daily_table, 'autorouter_llm_spend', 'double precision', TRUE), + (daily_table, 'autorouter_classifier_cost', 'double precision', TRUE), + (daily_table, 'autorouter_classifier_cost_recorded_requests', 'bigint', TRUE), + (daily_table, 'autorouter_estimated_requests', 'bigint', TRUE), + (daily_table, 'autorouter_estimated_actual_spend', 'double precision', TRUE), + (observation_table, 'daily_costs_publication', 'text', FALSE) + ) AS required(table_id, column_name, column_type, not_null) + LEFT JOIN pg_attribute a ON a.attrelid = required.table_id + AND a.attname = required.column_name AND a.attnum > 0 AND NOT a.attisdropped + LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum; + + IF invalid_columns THEN + RAISE EXCEPTION 'Daily auto-router coverage has incompatible column definitions in schema %', target_schema; + END IF; + + SELECT p.oid, + substring(obj_description(p.oid, 'pg_proc') FROM '^litellm:autorouter_daily_coverage:([0-9]{1,9})$')::integer, + p.prorettype = 'trigger'::regtype AND l.lanname = 'plpgsql' + AND p.provolatile = 'v' AND NOT p.prosecdef AND p.proconfig IS NULL + AND (p.prosrc = function_body OR COALESCE( + substring(obj_description(p.oid, 'pg_proc') FROM '^litellm:autorouter_daily_coverage:([0-9]{1,9})$')::integer, + 0 + ) > implementation_version) + INTO function_id, function_version, function_ready + FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + JOIN pg_language l ON l.oid = p.prolang + WHERE n.nspname = target_schema AND p.proname = 'litellm_update_daily_autorouter_coverage' + AND p.pronargs = 0; + + SELECT EXISTS ( + SELECT 1 FROM pg_trigger t + JOIN pg_attribute a ON a.attrelid = observation_table AND a.attname = 'publication' + AND a.attnum > 0 AND NOT a.attisdropped + WHERE t.tgrelid = observation_table AND t.tgname = 'litellm_update_daily_autorouter_coverage' + AND NOT t.tgisinternal AND t.tgenabled IN ('O', 'A') AND t.tgtype = 19 + AND t.tgattr::text = a.attnum::text AND t.tgnargs = 0 AND t.tgqual IS NULL + AND t.tgfoid = function_id + ) INTO trigger_ready; + + IF NOT missing_columns AND COALESCE(function_ready, FALSE) AND trigger_ready THEN + RETURN; + END IF; + IF function_version > implementation_version THEN + RAISE EXCEPTION 'Newer daily auto-router coverage is incompatible with this build in schema %', target_schema; + END IF; + IF attempt = 1 THEN + PERFORM pg_advisory_xact_lock(1279874117, 1145129292); + END IF; + END LOOP; + + IF missing_columns THEN + ALTER TABLE "LiteLLM_DailyUserSpend" + ADD COLUMN IF NOT EXISTS "autorouter_accounted_requests" BIGINT NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS "autorouter_requests" BIGINT NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS "autorouter_llm_spend" DOUBLE PRECISION NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS "autorouter_classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS "autorouter_classifier_cost_recorded_requests" BIGINT NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS "autorouter_estimated_requests" BIGINT NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS "autorouter_estimated_actual_spend" DOUBLE PRECISION NOT NULL DEFAULT 0; + + ALTER TABLE "LiteLLM_AutoRouterBaselineObservation" + ADD COLUMN IF NOT EXISTS "daily_costs_publication" TEXT; + END IF; + + IF NOT COALESCE(function_ready, FALSE) THEN + EXECUTE format( + 'CREATE OR REPLACE FUNCTION %I.litellm_update_daily_autorouter_coverage() ' + 'RETURNS TRIGGER AS %L LANGUAGE plpgsql VOLATILE SECURITY INVOKER', + target_schema, function_body + ); + EXECUTE format('ALTER FUNCTION %I.litellm_update_daily_autorouter_coverage() RESET ALL', target_schema); + EXECUTE format('COMMENT ON FUNCTION %I.litellm_update_daily_autorouter_coverage() IS %L', + target_schema, 'litellm:autorouter_daily_coverage:' || implementation_version::text); + END IF; + + IF NOT trigger_ready THEN + DROP TRIGGER IF EXISTS litellm_update_daily_autorouter_coverage ON "LiteLLM_AutoRouterBaselineObservation"; + CREATE TRIGGER litellm_update_daily_autorouter_coverage + BEFORE UPDATE OF publication ON "LiteLLM_AutoRouterBaselineObservation" + FOR EACH ROW EXECUTE PROCEDURE litellm_update_daily_autorouter_coverage(); + END IF; +END; +$install$; + +COMMIT; diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 85996430bc5..029e2fa7815 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -805,6 +805,13 @@ model LiteLLM_DailyUserSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + autorouter_accounted_requests BigInt @default(0) + autorouter_requests BigInt @default(0) + autorouter_llm_spend Float @default(0.0) + autorouter_classifier_cost Float @default(0.0) + autorouter_classifier_cost_recorded_requests BigInt @default(0) + autorouter_estimated_requests BigInt @default(0) + autorouter_estimated_actual_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1579,6 +1586,7 @@ model LiteLLM_AutoRouterBaselineObservation { revision BigInt data String publication String? + daily_costs_publication String? conflicted Boolean @default(false) @@index([scope, started_at, request_id], map: "idx_autorouter_baseline_event_order") diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 8a83c786e02..8189bff8924 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -1145,6 +1145,38 @@ class ProxyExtrasDBManager: prisma_env=prisma_env, ) + @staticmethod + def apply_autorouter_daily_coverage() -> None: + from litellm_proxy_extras.migration_lock import migration_environment + + package_dir: Final = Path(__file__).resolve().parent + migration: Final = package_dir / "migrations" / "20260923000000_add_daily_autorouter_costs" / "migration.sql" + for attempt in range(MAX_MIGRATE_DEPLOY_ATTEMPTS): + try: + prisma_toolchain.run_prisma( + [ + _get_prisma_command(), + "db", + "execute", + "--schema", + str(package_dir / "schema.prisma"), + "--file", + str(migration), + ], + timeout=prisma_command_timeout(), + env=migration_environment(_get_prisma_env()), + stdout=None, + stderr=None, + ) + return + except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: + if isinstance(error, OSError) or attempt + 1 == MAX_MIGRATE_DEPLOY_ATTEMPTS: + raise RuntimeError( + "Failed to install daily auto-router coverage; database setup cannot continue" + ) from error + logger.warning("Daily auto-router coverage installation attempt %s failed; retrying", attempt + 1) + time.sleep(random.uniform(0.5, 1.5)) + @staticmethod def setup_database( use_migrate: bool = False, use_v2_resolver: bool = False @@ -1170,6 +1202,7 @@ class ProxyExtrasDBManager: use_migrate=use_migrate, use_v2_resolver=use_v2_resolver ) if migrated: + ProxyExtrasDBManager.apply_autorouter_daily_coverage() ProxyExtrasDBManager.repair_invalid_indexes() ProxyExtrasDBManager.apply_replica_identity_full_if_requested() return migrated diff --git a/litellm/models/autorouter_session.py b/litellm/models/autorouter_session.py index ddce2b5ef81..019d3e8f711 100644 --- a/litellm/models/autorouter_session.py +++ b/litellm/models/autorouter_session.py @@ -34,15 +34,12 @@ class LiteLLM_AutoRouterSession(LiteLLMPydanticObjectBase): @property def baseline_model(self) -> str | None: - """The baseline most covered turns were priced against, or None when none were estimated. + """The baseline recorded by most session turns, including historical turns. A router reconfigured mid-session leaves turns priced against two baselines; the row keeps both - counts, and the label is the one that priced the most money-carrying turns rather than whatever the + counts, and the label is the one recorded by the most turns rather than whatever the router is configured with now. """ - if not self.savings_estimated_baseline_models: + if not self.baseline_models: return None - return max( - self.savings_estimated_baseline_models, - key=lambda model: (self.savings_estimated_baseline_models[model], model), - ) + return max(self.baseline_models, key=lambda model: (self.baseline_models[model], model)) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c7273738fd0..ae8f349036f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -5484,6 +5484,13 @@ class BaseDailySpendTransaction(TypedDict): # missing value to zero, so requiring it here would describe a shape the aggregation # is explicitly tested against. autorouter_savings_spend: NotRequired[float] + autorouter_accounted_requests: ReadOnly[NotRequired[int]] + autorouter_requests: ReadOnly[NotRequired[int]] + autorouter_llm_spend: ReadOnly[NotRequired[float]] + autorouter_classifier_cost: ReadOnly[NotRequired[float]] + autorouter_classifier_cost_recorded_requests: ReadOnly[NotRequired[int]] + autorouter_estimated_requests: ReadOnly[NotRequired[int]] + autorouter_estimated_actual_spend: ReadOnly[NotRequired[float]] # request level metrics spend: float diff --git a/litellm/proxy/client/cli/commands/statusline_script.py b/litellm/proxy/client/cli/commands/statusline_script.py index d16160b1ab8..ef6659f5234 100644 --- a/litellm/proxy/client/cli/commands/statusline_script.py +++ b/litellm/proxy/client/cli/commands/statusline_script.py @@ -70,6 +70,7 @@ class Session(NamedTuple): turns: int | None = None savings_estimated_turns: int | None = None savings_estimated_actual_spend: float | None = None + savings_estimated_baseline_spend: float | None = None class Credentials(NamedTuple): @@ -210,7 +211,8 @@ def _session_from_payload(payload: Mapping[str, object]) -> Session | None: router_name: Final = printable(payload.get("router_name")) last_model: Final = printable(payload.get("last_model")) spend: Final = payload.get("spend") - baseline_spend: Final = payload.get("savings_estimated_baseline_spend", payload.get("baseline_spend")) + baseline_spend: Final = payload.get("baseline_spend") + estimated_baseline_spend: Final = payload.get("savings_estimated_baseline_spend") turns: Final = payload.get("turns") estimated_turns: Final = payload.get("savings_estimated_turns") estimated_actual: Final = payload.get("savings_estimated_actual_spend") @@ -222,6 +224,12 @@ def _session_from_payload(payload: Mapping[str, object]) -> Session | None: not isinstance(baseline_spend, (int, float)) or isinstance(baseline_spend, bool) or not isfinite(baseline_spend) ): return None + if estimated_baseline_spend is not None and ( + not isinstance(estimated_baseline_spend, (int, float)) + or isinstance(estimated_baseline_spend, bool) + or not isfinite(estimated_baseline_spend) + ): + return None return Session( router_name=router_name, last_model=last_model, @@ -242,11 +250,14 @@ def _session_from_payload(payload: Mapping[str, object]) -> Session | None: and estimated_actual >= 0 else None ), + savings_estimated_baseline_spend=( + float(estimated_baseline_spend) if estimated_baseline_spend is not None else None + ), ) def cache_path(cache_dir: Path, credentials: Credentials, session_id: str) -> Path: - identity: Final = "\n".join((credentials.base_url, credentials.api_key, session_id)) + identity: Final = "\n".join(("v2", credentials.base_url, credentials.api_key, session_id)) return cache_dir / hashlib.sha256(identity.encode()).hexdigest() @@ -342,35 +353,41 @@ def render(model: str, session: Session | None, config_dir: Path, use_color: boo routed: Final = paint(BOLD, f"Routed to: {model}") if session is None: return routed - if session.savings_estimated_turns == 0 or session.baseline_spend is None: + covered_only: Final = session.baseline_spend is None + baseline_spend: Final = session.savings_estimated_baseline_spend if covered_only else session.baseline_spend + if baseline_spend is None or (covered_only and session.savings_estimated_turns == 0): return f"{routed}{SEPARATOR}Savings unavailable" - if session.baseline_model is None or session.baseline_spend <= 0: + if session.baseline_model is None or baseline_spend <= 0: return routed - if session.savings_estimated_turns is not None and ( - session.savings_estimated_actual_spend is None - or session.turns is None - or session.savings_estimated_turns > session.turns + if ( + covered_only + and session.savings_estimated_turns is not None + and ( + session.savings_estimated_actual_spend is None + or session.turns is None + or session.savings_estimated_turns > session.turns + ) ): return f"{routed}{SEPARATOR}Savings unavailable" compared_spend: Final = ( session.savings_estimated_actual_spend - if session.savings_estimated_turns is not None and session.savings_estimated_actual_spend is not None + if covered_only and session.savings_estimated_actual_spend is not None else session.spend ) coverage: Final = ( f"{SEPARATOR}{session.savings_estimated_turns} of {session.turns} turns estimated" - if session.savings_estimated_turns is not None + if covered_only and session.savings_estimated_turns is not None else "" ) reference: Final = baseline_label(session.baseline_model, config_dir) - pct: Final = round((session.baseline_spend - compared_spend) / session.baseline_spend * 100) + pct: Final = round((baseline_spend - compared_spend) / baseline_spend * 100) sign: Final = "-" if pct > 0 else "+" if pct < 0 else "" delta: Final = paint(LITELLM_COLOR, f"{sign}{abs(pct)}% vs {reference}") - peak: Final = max(compared_spend, session.baseline_spend) + peak: Final = max(compared_spend, baseline_spend) label_width: Final = max(_display_width(session.router_name), _display_width(reference)) rows: Final = ( (session.router_name, compared_spend, LITELLM_COLOR), - (reference, session.baseline_spend, BASELINE_COLOR), + (reference, baseline_spend, BASELINE_COLOR), ) lines: Final = ( f"{paint(DIM, label + ' ' * (label_width - _display_width(label)))} " diff --git a/litellm/proxy/db/autorouter_daily_spend.py b/litellm/proxy/db/autorouter_daily_spend.py new file mode 100644 index 00000000000..107622a5bcc --- /dev/null +++ b/litellm/proxy/db/autorouter_daily_spend.py @@ -0,0 +1,75 @@ +from math import isclose +from typing import Final, Literal + +from pydantic import BaseModel, ConfigDict + +DAILY_COSTS_COMPLETE_SQL: Final = """ + autorouter_accounted_requests = api_requests + AND autorouter_accounted_requests >= 0 + AND autorouter_requests BETWEEN 0 AND successful_requests + AND autorouter_classifier_cost_recorded_requests BETWEEN 0 AND autorouter_requests + AND autorouter_estimated_requests BETWEEN 0 AND autorouter_requests + AND (autorouter_savings_spend = 0 OR autorouter_estimated_requests > 0) +""" + +AUTOROUTER_DAILY_COSTS_SQL: Final = f""" +SELECT + COALESCE(SUM(autorouter_requests), 0)::bigint AS requests, + COALESCE(SUM(autorouter_llm_spend), 0)::float8 AS llm_spend, + COALESCE(SUM(autorouter_classifier_cost), 0)::float8 AS classifier_cost, + COALESCE(SUM(autorouter_classifier_cost_recorded_requests), 0)::bigint AS classifier_requests, + COALESCE(SUM(autorouter_estimated_requests), 0)::bigint AS estimated_requests, + COALESCE(SUM(autorouter_estimated_actual_spend), 0)::float8 AS estimated_actual_spend, + COALESCE(SUM(autorouter_savings_spend), 0)::float8 AS saved_spend, + COALESCE(BOOL_AND({DAILY_COSTS_COMPLETE_SQL}), TRUE) AS complete +FROM "LiteLLM_DailyUserSpend" +WHERE date >= $1::text AND date <= $2::text + AND ($3::text IS NULL OR api_key = $3::text) + AND ($4::text IS NULL OR user_id = $4::text) +""" + + +class AutoRouterDailyCosts(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + requests: int = 0 + llm_spend: float = 0.0 + classifier_cost: float = 0.0 + classifier_requests: int = 0 + estimated_requests: int = 0 + estimated_actual_spend: float = 0.0 + saved_spend: float = 0.0 + complete: bool = True + comparison_complete: bool = True + + def matches_savings(self, saved_spend: float) -> bool: + return self.complete and isclose(self.saved_spend, saved_spend, rel_tol=1e-9, abs_tol=1e-9) + + @property + def classifier_complete(self) -> bool: + return self.classifier_requests == self.requests + + @property + def coverage(self) -> Literal["complete", "partial", "unavailable"]: + if self.complete and self.classifier_complete: + return "complete" + return "partial" if self.requests > 0 else "unavailable" + + @property + def recorded_llm_spend(self) -> float | None: + return self.llm_spend if self.complete or self.requests > 0 else None + + @property + def recorded_classifier_cost(self) -> float | None: + return self.classifier_cost if self.classifier_complete and self.recorded_llm_spend is not None else None + + @property + def recorded_spend(self) -> float | None: + return self.llm_spend + self.classifier_cost if self.recorded_llm_spend is not None else None + + def baseline_spend(self, saved_spend: float) -> float | None: + if not self.matches_savings(saved_spend) or not self.classifier_complete or not self.comparison_complete: + return None + if self.requests > 0 and self.estimated_requests == 0: + return None + return self.estimated_actual_spend + saved_spend diff --git a/litellm/proxy/db/autorouter_historical_spend.py b/litellm/proxy/db/autorouter_historical_spend.py new file mode 100644 index 00000000000..5515668547a --- /dev/null +++ b/litellm/proxy/db/autorouter_historical_spend.py @@ -0,0 +1,181 @@ +import json +import sys +from contextlib import AbstractAsyncContextManager +from datetime import timedelta +from typing import TYPE_CHECKING, Final, Protocol, cast + +from litellm.constants import MAX_SPENDLOG_ROWS_TO_QUERY +from litellm.proxy.db.autorouter_daily_spend import DAILY_COSTS_COMPLETE_SQL, AutoRouterDailyCosts +from litellm.proxy.db.create_views import SupportsRawQueries + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + + +class _ReadTransactions(Protocol): + def tx(self, *, timeout: timedelta, max_wait: timedelta) -> AbstractAsyncContextManager[SupportsRawQueries]: ... + + +async def recover_daily_router_costs( + prisma_client: "PrismaClient", start_date: str, end_date: str, api_key: str | None, user_id: str | None +) -> AutoRouterDailyCosts | None: + from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING + + reader: Final = cast(_ReadTransactions, prisma_client.read_db) # cast-ok: untyped Prisma tx delegate + async with reader.tx(timeout=timedelta(seconds=3), max_wait=timedelta(seconds=1)) as transaction: + await transaction.execute_raw("SET TRANSACTION READ ONLY") + await transaction.execute_raw("SET LOCAL statement_timeout = 2000") + rows: Final = await transaction.query_raw( + AUTOROUTER_HISTORICAL_COSTS_SQL, start_date, end_date, api_key, user_id, json.dumps(ROUTE_ENDPOINT_MAPPING) + ) + return AutoRouterDailyCosts.model_validate(rows[0]) if rows else None + + +AUTOROUTER_HISTORICAL_COSTS_SQL: Final = f""" +WITH daily AS MATERIALIZED ( + SELECT + date, COALESCE(user_id, '') AS user_id, api_key, + COALESCE(model, '') AS model, + COALESCE(custom_llm_provider, '') AS custom_llm_provider, + COALESCE(mcp_namespaced_tool_name, '') AS mcp_namespaced_tool_name, + COALESCE(endpoint, '') AS endpoint, + SUM(api_requests)::bigint AS api_requests, + SUM(successful_requests)::bigint AS successful_requests, + SUM(failed_requests)::bigint AS failed_requests, + SUM(prompt_tokens)::bigint AS prompt_tokens, + SUM(completion_tokens)::bigint AS completion_tokens, + SUM(spend)::float8 AS spend, + SUM(autorouter_requests)::bigint AS requests, + SUM(autorouter_llm_spend)::float8 AS llm_spend, + SUM(autorouter_classifier_cost)::float8 AS classifier_cost, + SUM(autorouter_classifier_cost_recorded_requests)::bigint AS classifier_requests, + SUM(autorouter_estimated_requests)::bigint AS estimated_requests, + SUM(autorouter_estimated_actual_spend)::float8 AS estimated_actual_spend, + SUM(autorouter_savings_spend)::float8 AS saved_spend, + BOOL_AND({DAILY_COSTS_COMPLETE_SQL}) AS tracked_complete + FROM "LiteLLM_DailyUserSpend" + WHERE date >= $1::text AND date <= $2::text + AND ($3::text IS NULL OR api_key = $3::text) + AND ($4::text IS NULL OR user_id = $4::text) + GROUP BY 1, 2, 3, 4, 5, 6, 7 +), limited_logs AS MATERIALIZED ( + SELECT + to_char(logs."startTime", 'YYYY-MM-DD') AS date, + COALESCE(logs."user", '') AS user_id, logs.api_key, + COALESCE(logs.model, '') AS model, + COALESCE(logs.custom_llm_provider, '') AS custom_llm_provider, + COALESCE(logs.mcp_namespaced_tool_name, '') AS mcp_namespaced_tool_name, + COALESCE($5::jsonb ->> logs.call_type, '') AS endpoint, + logs.spend, logs.prompt_tokens, logs.completion_tokens, logs.status, + jsonb_build_object( + 'internal_call_origin', logs.metadata::jsonb -> 'internal_call_origin', + 'status', logs.metadata::jsonb -> 'status', + 'routing_decision', logs.metadata::jsonb -> 'routing_decision', + 'autorouter_savings', logs.metadata::jsonb -> 'autorouter_savings', + 'autorouter_savings_estimate', logs.metadata::jsonb -> 'autorouter_savings_estimate' + ) AS metadata + FROM "LiteLLM_SpendLogs" AS logs + WHERE logs."startTime" >= $1::text::timestamp + AND logs."startTime" < $2::text::timestamp + INTERVAL '1 day' + AND ($3::text IS NULL OR logs.api_key = $3::text) + AND ($4::text IS NULL OR logs."user" = $4::text) + AND EXISTS ( + SELECT 1 FROM daily + WHERE NOT daily.tracked_complete + AND daily.date = to_char(logs."startTime", 'YYYY-MM-DD') + AND daily.user_id = COALESCE(logs."user", '') + AND daily.api_key = logs.api_key + AND daily.model = COALESCE(logs.model, '') + AND daily.custom_llm_provider = COALESCE(logs.custom_llm_provider, '') + AND daily.mcp_namespaced_tool_name = COALESCE(logs.mcp_namespaced_tool_name, '') + AND daily.endpoint = COALESCE($5::jsonb ->> logs.call_type, '') + ) + LIMIT {MAX_SPENDLOG_ROWS_TO_QUERY + 1} +), log_facts AS ( + SELECT *, + COALESCE(metadata ->> 'internal_call_origin', '') = '' AS external, + COALESCE(metadata ->> 'status', '') <> 'failure' AS successful, + COALESCE(metadata ->> 'internal_call_origin', '') = '' + AND status = 'success' + AND jsonb_typeof(metadata -> 'routing_decision') = 'object' + AND metadata -> 'routing_decision' <> '{{}}'::jsonb AS routed, + CASE WHEN jsonb_typeof(metadata #> '{{routing_decision,classifier_cost}}') = 'number' + THEN CASE WHEN (metadata #>> '{{routing_decision,classifier_cost}}')::numeric + BETWEEN -{sys.float_info.max} AND {sys.float_info.max} + THEN (metadata #>> '{{routing_decision,classifier_cost}}')::float8 END END AS classifier, + CASE WHEN jsonb_typeof(metadata -> 'autorouter_savings') = 'number' + THEN CASE WHEN (metadata ->> 'autorouter_savings')::numeric + BETWEEN -{sys.float_info.max} AND {sys.float_info.max} + THEN (metadata ->> 'autorouter_savings')::float8 END END AS recorded_savings, + metadata -> 'autorouter_savings_estimate' AS estimate + FROM limited_logs +), classified_logs AS ( + SELECT *, + recorded_savings IS NOT NULL AND ( + estimate IS NULL OR estimate = 'null'::jsonb OR ( + jsonb_typeof(estimate -> 'version') = 'number' + AND estimate ->> 'version' IN ('1', '2', '3') + AND estimate ->> 'status' = 'estimated' + ) + ) AS estimated, + jsonb_typeof(estimate -> 'version') = 'number' + AND estimate ->> 'version' IN ('1', '2', '3') + AND estimate ->> 'status' = 'unknown' AS unknown + FROM log_facts +), log_totals AS ( + SELECT date, user_id, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint, + COUNT(*) FILTER (WHERE external)::bigint AS api_requests, + COUNT(*) FILTER (WHERE external AND successful)::bigint AS successful_requests, + COUNT(*) FILTER (WHERE external AND NOT successful)::bigint AS failed_requests, + SUM(prompt_tokens)::bigint AS prompt_tokens, + SUM(completion_tokens)::bigint AS completion_tokens, + SUM(spend)::float8 AS spend, + COUNT(*) FILTER (WHERE routed)::bigint AS requests, + COALESCE(SUM(spend) FILTER (WHERE routed), 0)::float8 AS llm_spend, + COALESCE(SUM(classifier) FILTER (WHERE routed), 0)::float8 AS classifier_cost, + COUNT(*) FILTER (WHERE routed AND classifier IS NOT NULL)::bigint AS classifier_requests, + COUNT(*) FILTER (WHERE routed AND estimated)::bigint AS estimated_requests, + COALESCE(SUM(spend + COALESCE(classifier, 0)) FILTER (WHERE routed AND estimated), 0)::float8 + AS estimated_actual_spend, + COALESCE(SUM(recorded_savings) FILTER (WHERE routed AND estimated), 0)::float8 AS saved_spend, + COALESCE(BOOL_AND(COALESCE(estimated OR unknown, FALSE)) FILTER (WHERE routed), TRUE) + AS comparison_complete + FROM classified_logs + GROUP BY 1, 2, 3, 4, 5, 6, 7 +), reconciled AS ( + SELECT daily.*, logs.requests AS recovered_requests, logs.llm_spend AS recovered_llm_spend, + logs.classifier_cost AS recovered_classifier_cost, + logs.classifier_requests AS recovered_classifier_requests, + logs.estimated_requests AS recovered_estimated_requests, + logs.estimated_actual_spend AS recovered_estimated_actual_spend, + COALESCE(logs.comparison_complete + AND ABS(logs.saved_spend - daily.saved_spend) + <= GREATEST(1e-9, ABS(daily.saved_spend) * 1e-9), FALSE) AS recovered_comparison_complete, + COALESCE((SELECT COUNT(*) FROM limited_logs) <= {MAX_SPENDLOG_ROWS_TO_QUERY} + AND logs.api_requests = daily.api_requests + AND logs.successful_requests = daily.successful_requests + AND logs.failed_requests = daily.failed_requests + AND logs.prompt_tokens = daily.prompt_tokens + AND logs.completion_tokens = daily.completion_tokens + AND ABS(logs.spend - daily.spend) <= GREATEST(1e-9, ABS(daily.spend) * 1e-9) + AND (daily.saved_spend = 0 OR logs.requests > 0), FALSE) AS recovered + FROM daily LEFT JOIN log_totals AS logs + USING (date, user_id, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint) +) +SELECT + COALESCE(SUM(CASE WHEN recovered THEN recovered_requests ELSE requests END), 0)::bigint AS requests, + COALESCE(SUM(CASE WHEN recovered THEN recovered_llm_spend ELSE llm_spend END), 0)::float8 AS llm_spend, + COALESCE(SUM(CASE WHEN recovered THEN recovered_classifier_cost ELSE classifier_cost END), 0)::float8 + AS classifier_cost, + COALESCE(SUM(CASE WHEN recovered THEN recovered_classifier_requests ELSE classifier_requests END), 0)::bigint + AS classifier_requests, + COALESCE(SUM(CASE WHEN recovered THEN recovered_estimated_requests ELSE estimated_requests END), 0)::bigint + AS estimated_requests, + COALESCE(SUM(CASE WHEN recovered THEN recovered_estimated_actual_spend ELSE estimated_actual_spend END), 0)::float8 + AS estimated_actual_spend, + COALESCE(SUM(saved_spend), 0)::float8 AS saved_spend, + COALESCE(BOOL_AND(tracked_complete OR recovered), TRUE) AS complete, + COALESCE(BOOL_AND(tracked_complete OR (recovered AND recovered_comparison_complete)), TRUE) + AS comparison_complete +FROM reconciled +""" diff --git a/litellm/proxy/db/daily_spend_bulk_upsert.py b/litellm/proxy/db/daily_spend_bulk_upsert.py index eb130a5196f..ae9c35c55c5 100644 --- a/litellm/proxy/db/daily_spend_bulk_upsert.py +++ b/litellm/proxy/db/daily_spend_bulk_upsert.py @@ -99,15 +99,34 @@ _SPEND_COLUMNS: Final = ( "gateway_injected_caching_savings_spend", "autorouter_savings_spend", ) +_AUTOROUTER_COUNTER_COLUMNS: Final = ( + "autorouter_accounted_requests", + "autorouter_requests", + "autorouter_classifier_cost_recorded_requests", + "autorouter_estimated_requests", +) +_AUTOROUTER_SPEND_COLUMNS: Final = ( + "autorouter_llm_spend", + "autorouter_classifier_cost", + "autorouter_estimated_actual_spend", +) _CASTS: Final[Mapping[str, str]] = MappingProxyType( { - **{column: "bigint" for column in _COUNTER_COLUMNS}, - **{column: "double precision" for column in _SPEND_COLUMNS}, + **{column: "bigint" for column in (*_COUNTER_COLUMNS, *_AUTOROUTER_COUNTER_COLUMNS)}, + **{column: "double precision" for column in (*_SPEND_COLUMNS, *_AUTOROUTER_SPEND_COLUMNS)}, } ) +def _counter_columns(table: DailySpendTable) -> tuple[str, ...]: + return (*_COUNTER_COLUMNS, *(_AUTOROUTER_COUNTER_COLUMNS if table.name == "LiteLLM_DailyUserSpend" else ())) + + +def _spend_columns(table: DailySpendTable) -> tuple[str, ...]: + return (*_SPEND_COLUMNS, *(_AUTOROUTER_SPEND_COLUMNS if table.name == "LiteLLM_DailyUserSpend" else ())) + + def _quoted(columns: Sequence[str]) -> str: return ", ".join(f'"{column}"' for column in columns) @@ -129,13 +148,13 @@ def conflict_key(table: DailySpendTable, transaction: SpendRow) -> tuple[str, .. return tuple(_as_text(transaction.get(column)) for column in (table.entity_id_column, *_KEY_COLUMNS)) -def _merge(group: Sequence[SpendRow]) -> SpendRow: +def _merge(table: DailySpendTable, group: Sequence[SpendRow]) -> SpendRow: if len(group) == 1: return group[0] return { **group[0], - **{column: sum(_as_int(row.get(column)) for row in group) for column in _COUNTER_COLUMNS}, - **{column: sum(_as_float(row.get(column)) for row in group) for column in _SPEND_COLUMNS}, + **{column: sum(_as_int(row.get(column)) for row in group) for column in _counter_columns(table)}, + **{column: sum(_as_float(row.get(column)) for row in group) for column in _spend_columns(table)}, } @@ -152,7 +171,9 @@ def merge_by_conflict_key( key keeps concurrent writers taking row locks in the same sequence. """ ordered: Final = sorted(transactions, key=lambda transaction: conflict_key(table, transaction)) - return tuple((key, _merge(tuple(group))) for key, group in groupby(ordered, key=lambda t: conflict_key(table, t))) + return tuple( + (key, _merge(table, tuple(group))) for key, group in groupby(ordered, key=lambda t: conflict_key(table, t)) + ) def _row_params( @@ -165,8 +186,8 @@ def _row_params( str(uuid.uuid4()), *key, None if transaction.get("model_group") is None else _as_text(transaction.get("model_group")), - *(_as_int(transaction.get(column)) for column in _COUNTER_COLUMNS), - *(_as_float(transaction.get(column)) for column in _SPEND_COLUMNS), + *(_as_int(transaction.get(column)) for column in _counter_columns(table)), + *(_as_float(transaction.get(column)) for column in _spend_columns(table)), *((None if request_id is None else _as_text(request_id),) if table.carries_request_id else ()), ) @@ -177,8 +198,8 @@ def _insert_columns(table: DailySpendTable) -> tuple[str, ...]: table.entity_id_column, *_KEY_COLUMNS, "model_group", - *_COUNTER_COLUMNS, - *_SPEND_COLUMNS, + *_counter_columns(table), + *_spend_columns(table), *(("request_id",) if table.carries_request_id else ()), ) @@ -201,7 +222,7 @@ def build_bulk_upsert( ) increments: Final = ", ".join( f'"{column}" = {quoted_table}."{column}" + EXCLUDED."{column}"' - for column in (*_COUNTER_COLUMNS, *_SPEND_COLUMNS) + for column in (*_counter_columns(table), *_spend_columns(table)) ) # request_id names one arbitrary contributing request, so an entry carrying none must # not blank out the one already recorded. diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index e38214c98a6..6f6c22c7418 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -74,9 +74,11 @@ from litellm.proxy.spend_tracking.compression_savings import ( extract_compression_saved_tokens, ) from litellm.proxy.spend_tracking.savings import ( + classifier_cost_from_decision, compute_savings_spend, extract_cache_creation_tokens, extract_cache_read_tokens, + known_autorouter_savings, marks_gateway_injection, ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error @@ -2621,6 +2623,29 @@ class DBSpendUpdateWriter: endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None) is_internal_call: Final = bool(_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) + routing_decision: Final = _metadata.get("routing_decision") + is_routed_request: Final = ( + not is_internal_call + and payload.get("status") == "success" + and isinstance(routing_decision, Mapping) + and bool(routing_decision) + ) + classifier_cost: Final = classifier_cost_from_decision(routing_decision) if is_routed_request else None + known_savings: Final = ( + known_autorouter_savings( + model=payload.get("model"), + custom_llm_provider=payload.get("custom_llm_provider"), + routing_decision=routing_decision, + usage_object=usage_obj, + model_id=payload.get("model_id"), + llm_router=get_llm_router, + cost_breakdown=_metadata.get("cost_breakdown"), + recorded_autorouter_savings=_metadata.get("autorouter_savings"), + recorded_autorouter_savings_estimate=_metadata.get("autorouter_savings_estimate"), + ) + if is_routed_request + else None + ) cache_read_input_tokens: Final = extract_cache_read_tokens(usage_obj) compression_saved_tokens: Final = extract_compression_saved_tokens(_metadata) savings_spend: Final = compute_savings_spend( @@ -2665,6 +2690,15 @@ class DBSpendUpdateWriter: prompt_caching_savings_spend=savings_spend.prompt_caching, gateway_injected_caching_savings_spend=savings_spend.gateway_injected_caching, autorouter_savings_spend=0.0 if is_internal_call else savings_spend.autorouter, + autorouter_accounted_requests=0 if is_internal_call else 1, + autorouter_requests=int(is_routed_request), + autorouter_llm_spend=payload["spend"] if is_routed_request else 0.0, + autorouter_classifier_cost=classifier_cost or 0.0, + autorouter_classifier_cost_recorded_requests=int(classifier_cost is not None), + autorouter_estimated_requests=int(known_savings is not None), + autorouter_estimated_actual_spend=( + payload["spend"] + (classifier_cost or 0.0) if known_savings is not None else 0.0 + ), total_response_time_ms=timed_duration_ms or 0, timed_requests=0 if timed_duration_ms is None else 1, ) diff --git a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py index c6381cd070b..afee9cf60a8 100644 --- a/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py +++ b/litellm/proxy/db/db_transaction_queue/daily_spend_update_queue.py @@ -149,6 +149,27 @@ class DailySpendUpdateQueue(BaseUpdateQueue): daily_transaction["timed_requests"] = ( payload.get("timed_requests", 0) or 0 ) + daily_transaction.get("timed_requests", 0) + aggregated_daily_spend_update_transactions[_key] = { + **daily_transaction, + "autorouter_accounted_requests": daily_transaction.get("autorouter_accounted_requests", 0) + + payload.get("autorouter_accounted_requests", 0), + "autorouter_requests": daily_transaction.get("autorouter_requests", 0) + + payload.get("autorouter_requests", 0), + "autorouter_llm_spend": daily_transaction.get("autorouter_llm_spend", 0.0) + + payload.get("autorouter_llm_spend", 0.0), + "autorouter_classifier_cost": daily_transaction.get("autorouter_classifier_cost", 0.0) + + payload.get("autorouter_classifier_cost", 0.0), + "autorouter_classifier_cost_recorded_requests": daily_transaction.get( + "autorouter_classifier_cost_recorded_requests", 0 + ) + + payload.get("autorouter_classifier_cost_recorded_requests", 0), + "autorouter_estimated_requests": daily_transaction.get("autorouter_estimated_requests", 0) + + payload.get("autorouter_estimated_requests", 0), + "autorouter_estimated_actual_spend": daily_transaction.get( + "autorouter_estimated_actual_spend", 0.0 + ) + + payload.get("autorouter_estimated_actual_spend", 0.0), + } else: aggregated_daily_spend_update_transactions[_key] = deepcopy(payload) diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py index 0524d015047..3f96889816b 100644 --- a/litellm/proxy/db/prisma_client.py +++ b/litellm/proxy/db/prisma_client.py @@ -949,6 +949,7 @@ class PrismaManager: prisma_command_timeout, run_prisma, ) + from litellm_proxy_extras.utils import ProxyExtrasDBManager except ImportError as e: verbose_proxy_logger.error("\x1b[1;31mLiteLLM: Failed to import proxy extras. Got %s\x1b[0m", e) return False @@ -967,6 +968,7 @@ class PrismaManager: stdout=None, stderr=None, ) + ProxyExtrasDBManager.apply_autorouter_daily_coverage() PrismaManager._apply_replica_identity_full_if_requested() return True except subprocess.TimeoutExpired as e: diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 2ae8639fe61..b1d3c46ff37 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -31,6 +31,8 @@ from litellm.proxy.auth.auth_checks import ( can_key_call_resolved_model, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.db.autorouter_daily_spend import AUTOROUTER_DAILY_COSTS_SQL, AutoRouterDailyCosts +from litellm.proxy.db.autorouter_historical_spend import recover_daily_router_costs from litellm.proxy.db.autorouter_session_rollup import ( AUTOROUTER_BENCHMARKS_SQL, bounded_session_id, @@ -686,10 +688,31 @@ def _savings_cohort( return saved_spend, actual_spend + saved_spend +def _recorded_baseline_spend(estimated_turns: int, actual_spend: float, saved_spend: float) -> float | None: + if estimated_turns == 0 and saved_spend == 0: + return None + return actual_spend + saved_spend + + +async def _daily_router_costs( + prisma_client: "PrismaClient", start_date: str, end_date: str, api_key: str | None, user_id: str | None +) -> AutoRouterDailyCosts: + rows: Final = await _query_raw(prisma_client, AUTOROUTER_DAILY_COSTS_SQL, start_date, end_date, api_key, user_id) + recorded: Final = AutoRouterDailyCosts.model_validate(rows[0]) if rows else AutoRouterDailyCosts() + if recorded.complete: + return recorded + try: + recovered: Final = await recover_daily_router_costs(prisma_client, start_date, end_date, api_key, user_id) + return recovered if recovered is not None else recorded + except Exception: # noqa: BLE001 # optional recovery must preserve durable daily savings and costs + verbose_proxy_logger.warning("Auto-router historical cost recovery unavailable; using recorded daily costs") + return recorded + + def _benchmark_totals(row: _SessionAggRow) -> AutoRouterBenchmarkTotals: return_misses: Final = row.return_turns - row.return_hits - saved_spend, baseline_spend = _savings_cohort( - row.turns, row.savings_estimated_turns, row.savings_estimated_actual_spend, row.savings_estimated_saved_spend + baseline_spend: Final = ( + _recorded_baseline_spend(row.savings_estimated_turns, row.spend, row.saved_spend) if row.turns else 0.0 ) sessions: Final = row.sessions return AutoRouterBenchmarkTotals( @@ -699,15 +722,16 @@ def _benchmark_totals(row: _SessionAggRow) -> AutoRouterBenchmarkTotals: avg_session_seconds=row.session_seconds / sessions if sessions else 0.0, avg_tokens_per_session=row.total_tokens / sessions if sessions else 0.0, spend=row.spend, + llm_spend=row.spend - row.classifier_cost if row.classifier_cost_recorded_turns == row.turns else None, + cost_coverage="complete" if row.classifier_cost_recorded_turns == row.turns else "partial", + cost_requests=row.turns, savings_estimated_turns=row.savings_estimated_turns, savings_estimated_actual_spend=row.savings_estimated_actual_spend, - saved_spend=saved_spend, + saved_spend=row.saved_spend, classifier_cost=row.classifier_cost if row.classifier_cost_recorded_turns == row.turns else None, baseline_spend=baseline_spend, - saved_pct=_pct(saved_spend, baseline_spend) if saved_spend is not None and baseline_spend is not None else None, - saved_per_session=(row.savings_estimated_saved_spend / sessions if sessions else 0.0) - if row.savings_estimated_turns == row.turns - else None, + saved_pct=_pct(row.saved_spend, baseline_spend) if baseline_spend is not None else None, + saved_per_session=row.saved_spend / sessions if sessions else 0.0, cache=AutoRouterCacheStats( coverage_pct=_pct(row.covered_turns, row.turns), hit_rate_pct=_pct(row.cache_hits, row.covered_turns), @@ -736,6 +760,9 @@ def _benchmark_group(row: _SessionAggRow) -> AutoRouterBenchmarkGroup: avg_session_seconds=totals.avg_session_seconds, avg_tokens_per_session=totals.avg_tokens_per_session, spend=totals.spend, + llm_spend=totals.llm_spend, + cost_coverage=totals.cost_coverage, + cost_requests=totals.cost_requests, saved_spend=totals.saved_spend, savings_estimated_turns=totals.savings_estimated_turns, savings_estimated_actual_spend=totals.savings_estimated_actual_spend, @@ -847,12 +874,12 @@ async def get_auto_router_benchmarks( Benchmarks for the auto-router dashboard: session shape, savings against the configured baseline, and prompt-caching behaviour bucketed by what the router did. - Reads session rollups folded once per request at spend-write time, so this endpoint - never scans LiteLLM_SpendLogs. A user filter selects only turns attributed to that - internal user when written; older key-only history remains outside user views. A session - is in the window when it overlaps it: its last turn is on or after start_date and its first turn is on or before - end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is - over that bucket's turns. + Total savings use the same request-date daily aggregation as Overall cost optimization, + including recorded history and requests without session IDs. Costs use matching request + dates with explicit coverage for history predating daily router cost tracking. Session + statistics and per-router groups cover whole sessions that overlap the window. + A user filter uses the user recorded when each row was written. + Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is over its turns. The rollup supplies the measures, never the list. Which routers appear comes from the model registry, so one shows up as soon as it is configured and reads zero until it @@ -874,6 +901,20 @@ async def get_auto_router_benchmarks( if end_day < start_day: raise HTTPException(status_code=400, detail="end_date must not be earlier than start_date") + from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity_aggregated + + daily: Final = await get_daily_activity_aggregated( + prisma_client=prisma_client, + table_name="litellm_dailyuserspend", + entity_id_field="user_id", + entity_id=user_id, + entity_metadata_field=None, + start_date=start_day.strftime("%Y-%m-%d"), + end_date=end_day.strftime("%Y-%m-%d"), + model=None, + api_key=api_key, + ) + raw_rows: Final = await _query_raw( prisma_client, AUTOROUTER_BENCHMARKS_SQL, @@ -887,11 +928,36 @@ async def get_auto_router_benchmarks( *(_benchmark_group(row) for row in rows), *_idle_router_groups(llm_router, frozenset((row.router_name, row.router_type) for row in rows)), ) + session_totals: Final = _benchmark_totals(_summed_agg_row(rows)) + costs: Final = await _daily_router_costs( + prisma_client, + start_day.strftime("%Y-%m-%d"), + end_day.strftime("%Y-%m-%d"), + api_key, + user_id, + ) + saved_spend: Final = daily.metadata.total_autorouter_savings_spend + baseline_spend: Final = costs.baseline_spend(saved_spend) return AutoRouterBenchmarksResponse( start_date=start_day.strftime("%Y-%m-%d"), end_date=end_day.strftime("%Y-%m-%d"), routers_in_scope=len(groups), - totals=_benchmark_totals(_summed_agg_row(rows)), + totals=session_totals.model_copy( + update=MappingProxyType( + { + "saved_spend": saved_spend, + "spend": costs.recorded_spend, + "llm_spend": costs.recorded_llm_spend, + "classifier_cost": costs.recorded_classifier_cost, + "cost_coverage": costs.coverage, + "cost_requests": costs.requests if costs.complete else None, + "savings_estimated_turns": costs.estimated_requests, + "savings_estimated_actual_spend": costs.estimated_actual_spend, + "baseline_spend": baseline_spend, + "saved_pct": _pct(saved_spend, baseline_spend) if baseline_spend is not None else None, + } + ) + ), groups=groups, ) @@ -927,7 +993,7 @@ async def get_auto_router_session( raise HTTPException( status_code=404, detail=f"No auto-routed turns recorded for session {session_id!r} under this key" ) - saved_spend, baseline_spend = _savings_cohort( + _, estimated_baseline_spend = _savings_cohort( row.turns, row.savings_estimated_turns, row.savings_estimated_actual_spend, row.savings_estimated_saved_spend ) return AutoRouterSessionResponse( @@ -939,11 +1005,11 @@ async def get_auto_router_session( spend=row.spend, savings_estimated_turns=row.savings_estimated_turns, savings_estimated_actual_spend=row.savings_estimated_actual_spend, - saved_spend=saved_spend, - baseline_spend=baseline_spend if row.savings_estimated_turns == row.turns else None, - savings_estimated_baseline_spend=baseline_spend, + saved_spend=row.saved_spend, + baseline_spend=_recorded_baseline_spend(row.savings_estimated_turns, row.spend, row.saved_spend), + savings_estimated_baseline_spend=estimated_baseline_spend, baseline_model=row.baseline_model, - baseline_models=row.savings_estimated_baseline_models, + baseline_models=row.baseline_models, ) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 85996430bc5..029e2fa7815 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -805,6 +805,13 @@ model LiteLLM_DailyUserSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + autorouter_accounted_requests BigInt @default(0) + autorouter_requests BigInt @default(0) + autorouter_llm_spend Float @default(0.0) + autorouter_classifier_cost Float @default(0.0) + autorouter_classifier_cost_recorded_requests BigInt @default(0) + autorouter_estimated_requests BigInt @default(0) + autorouter_estimated_actual_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1579,6 +1586,7 @@ model LiteLLM_AutoRouterBaselineObservation { revision BigInt data String publication String? + daily_costs_publication String? conflicted Boolean @default(false) @@index([scope, started_at, request_id], map: "idx_autorouter_baseline_event_order") diff --git a/litellm/proxy/spend_tracking/savings.py b/litellm/proxy/spend_tracking/savings.py index fbcf9c78d3e..57a5e770a81 100644 --- a/litellm/proxy/spend_tracking/savings.py +++ b/litellm/proxy/spend_tracking/savings.py @@ -628,6 +628,42 @@ def prompt_caching_savings_for_request( return _prompt_caching_savings(request_pricing[1], request_pricing[0], usage_object, cost_breakdown, billed_at) +def known_autorouter_savings( + model: str | None, + custom_llm_provider: str | None, + routing_decision: Mapping[str, object] | None, + usage_object: Mapping[str, object] | None, + model_id: str | None = None, + llm_router: "Callable[[], Router | None] | None" = None, + cost_breakdown: Mapping[str, object] | None = None, + recorded_autorouter_savings: object = None, + recorded_autorouter_savings_estimate: Mapping[str, object] | None = None, +) -> float | None: + recorded: Final = ( + recorded_estimated_autorouter_savings( + MappingProxyType( + { + "autorouter_savings": recorded_autorouter_savings, + "autorouter_savings_estimate": recorded_autorouter_savings_estimate, + } + ) + ) + if recorded_autorouter_savings_estimate is not None + else _numeric_savings(recorded_autorouter_savings) + ) + if recorded is not None or recorded_autorouter_savings_estimate is not None: + return recorded + return autorouter_savings_for_request( + model=model, + custom_llm_provider=custom_llm_provider, + routing_decision=routing_decision, + usage_object=usage_object, + model_id=model_id, + llm_router=llm_router, + cost_breakdown=cost_breakdown, + ) + + def compute_savings_spend( model: str | None, custom_llm_provider: str | None, @@ -697,32 +733,16 @@ def compute_savings_spend( prompt_caching: Final = _prompt_caching_savings(pricing, provider, usage_object, cost_breakdown, billed_at) or 0.0 gateway_injected_caching: Final = prompt_caching if gateway_injected_cache else 0.0 - # The figure the logging path recorded wins, before the usage gate on purpose: a row - # whose usage no longer parses still carries the number computed when it did. - recorded_savings: Final = ( - recorded_estimated_autorouter_savings( - MappingProxyType( - { - "autorouter_savings": recorded_autorouter_savings, - "autorouter_savings_estimate": recorded_autorouter_savings_estimate, - } - ) - ) - if recorded_autorouter_savings_estimate is not None - else _numeric_savings(recorded_autorouter_savings) - ) - autorouter: Final = ( - recorded_savings - if recorded_savings is not None or recorded_autorouter_savings_estimate is not None - else autorouter_savings_for_request( - model=model, - custom_llm_provider=custom_llm_provider, - routing_decision=routing_decision, - usage_object=usage_object, - model_id=model_id, - llm_router=llm_router, - cost_breakdown=cost_breakdown, - ) + autorouter: Final = known_autorouter_savings( + model=model, + custom_llm_provider=custom_llm_provider, + routing_decision=routing_decision, + usage_object=usage_object, + model_id=model_id, + llm_router=llm_router, + cost_breakdown=cost_breakdown, + recorded_autorouter_savings=recorded_autorouter_savings, + recorded_autorouter_savings_estimate=recorded_autorouter_savings_estimate, ) return SavingsSpend( compression=compression, diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 334ca0dfb08..4b2731b2088 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -211,37 +211,56 @@ class AutoRouterCacheStats(BaseModel): class AutoRouterBenchmarkTotals(BaseModel): - """Session-shape and savings aggregates over auto-routed traffic in the window.""" + """Request-date savings totals with statistics for whole sessions overlapping the window.""" sessions: int turns: int avg_turns_per_session: float avg_session_seconds: float avg_tokens_per_session: float - spend: float = Field(description="What the routed traffic actually cost") + spend: float | None = Field( + description="Actual auto-router cost including classification. Totals use request dates; groups cover whole " + "overlapping sessions. Partial recorded history is identified by cost_coverage" + ) classifier_cost: float | None = Field( - description="Recorded LLM classifier cost already included in spend; null when any session turns predate " - "subtotal recording, and zero for an empty window" + description="Classifier charges included in actual cost; null when any contributing request lacks the subtotal" + ) + llm_spend: float | None = Field(default=None, description="Routed LLM cost excluding classifier charges") + cost_coverage: Literal["complete", "partial", "unavailable"] = Field( + default="unavailable", description="Whether recorded costs cover the entire selected population" + ) + cost_requests: int | None = Field( + default=None, + description="Number of requests in the cost population; null when historical coverage is incomplete", ) savings_estimated_turns: int = Field( - description="Turns covered by the current savings estimator; legacy estimates are excluded" + description="Requests with an available savings comparison. Totals use request dates and groups use session " + "coverage. Historical savings remain included even when their coverage was not tracked" ) savings_estimated_actual_spend: float = Field( - description="Actual spend, including classifier cost, for covered turns only" + description="Actual spend including classification for requests covered by the savings comparison" ) saved_spend: float | None = Field( - description="Signed savings for covered turns only; null when traffic has no current estimates" + description="Signed recorded savings, including historical and current estimates once. Totals use the same " + "request-date daily source as Overall cost optimization, including requests without sessions. Per-router " + "groups cover whole overlapping sessions. Turns without an estimate add no savings" + ) + baseline_spend: float | None = Field( + description="Matching actual spend plus recorded savings. Daily totals compare estimated requests only; " + "null when historical cost coverage or the savings estimate is unavailable" + ) + saved_pct: float | None = Field( + description="Recorded savings divided by the matching estimated baseline; null when that baseline is unavailable" ) - baseline_spend: float | None = Field(description="Estimated single-model cost for covered turns only") - saved_pct: float | None = Field(description="Covered savings over covered baseline spend, as a percentage") saved_per_session: float | None = Field( - description="Average session savings; unavailable unless every turn is covered" + description="Recorded savings in whole overlapping sessions divided by their session count, independent " + "of request-date daily savings totals" ) cache: AutoRouterCacheStats class AutoRouterBenchmarkGroup(AutoRouterBenchmarkTotals): - """One auto-router's slice of the benchmarks.""" + """One auto-router's whole overlapping sessions, separate from request-date daily totals.""" router_name: str = Field(description="The auto-router alias requests were sent to") router_type: str = Field(description="complexity, adaptive or quality") @@ -273,27 +292,29 @@ class AutoRouterSessionResponse(BaseModel): savings_estimated_actual_spend: float = Field( description="Actual spend, including classifier cost, for covered turns only" ) - saved_spend: float | None = Field(description="Estimated savings for covered turns only, net of classifier cost") + saved_spend: float | None = Field( + description="Recorded signed savings, retaining historical estimates and adding current estimates once" + ) baseline_spend: float | None = Field( - description="Estimated single-model cost; unavailable unless every turn is covered" + description="Actual session spend plus recorded savings; turns without an estimate add no savings" ) savings_estimated_baseline_spend: float | None = Field( description="Estimated single-model cost for covered turns only" ) baseline_model: str | None = Field( - description="The savings baseline most covered turns were priced against, recorded turn by " + description="The savings baseline recorded by most session turns, including historical turns, recorded turn by " "turn, so it still names the counterfactual after the router is reconfigured or removed. None when no " "turn recorded one: rows from before the baseline was recorded, and adaptive and quality routers, " "which derive no baseline and so report no savings" ) baseline_models: Mapping[str, int] = Field( - description="Covered turns priced against each baseline model; more than one entry means the router's " - "baseline changed mid-session and baseline_spend mixes both" + description="Session turns recording each baseline model, including historical turns; more than one entry " + "means the router's baseline changed mid-session. These counts do not imply savings-estimate coverage" ) class AutoRouterBenchmarksResponse(BaseModel): - """Benchmarks for the auto-router dashboard, aggregated from the per-session rollup.""" + """Daily recorded savings with tracked-session breakdowns for the auto-router dashboard.""" start_date: str = Field(description="Window start day, YYYY-MM-DD UTC, inclusive") end_date: str = Field(description="Window end day, YYYY-MM-DD UTC, inclusive") diff --git a/schema.prisma b/schema.prisma index 85996430bc5..029e2fa7815 100644 --- a/schema.prisma +++ b/schema.prisma @@ -805,6 +805,13 @@ model LiteLLM_DailyUserSpend { prompt_caching_savings_spend Float @default(0.0) gateway_injected_caching_savings_spend Float @default(0.0) autorouter_savings_spend Float @default(0.0) + autorouter_accounted_requests BigInt @default(0) + autorouter_requests BigInt @default(0) + autorouter_llm_spend Float @default(0.0) + autorouter_classifier_cost Float @default(0.0) + autorouter_classifier_cost_recorded_requests BigInt @default(0) + autorouter_estimated_requests BigInt @default(0) + autorouter_estimated_actual_spend Float @default(0.0) spend Float @default(0.0) api_requests BigInt @default(0) successful_requests BigInt @default(0) @@ -1579,6 +1586,7 @@ model LiteLLM_AutoRouterBaselineObservation { revision BigInt data String publication String? + daily_costs_publication String? conflicted Boolean @default(false) @@index([scope, started_at, request_id], map: "idx_autorouter_baseline_event_order") diff --git a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py index bb329264a11..10619468d70 100644 --- a/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py +++ b/tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py @@ -3,6 +3,7 @@ import os import re import sys from pathlib import Path +from typing import Final import pytest @@ -740,16 +741,28 @@ ERROR: relation "SomeTable" already exists ), ), ) -def test_v2_migrations_use_the_direct_connection_with_the_runtime_schema(pooled, direct, expected): +def test_v2_migrations_use_the_direct_connection_with_the_runtime_schema( + monkeypatch: pytest.MonkeyPatch, pooled: str, direct: str | None, expected: str +) -> None: + from unittest.mock import Mock + + import litellm_proxy_extras.utils as utils_module from litellm_proxy_extras.migration_lock import migration_environment - environment = {"DATABASE_URL": pooled, "PRISMA_OFFLINE_MODE": "true"} - configured = {**environment, **({"DIRECT_URL": direct} if direct else {})} - migrated = migration_environment(configured) + environment: Final = {"DATABASE_URL": pooled, "PRISMA_OFFLINE_MODE": "true"} + configured: Final = {**environment, **({"DIRECT_URL": direct} if direct else {})} + migrated: Final = migration_environment(configured) + run_prisma: Final = Mock() + monkeypatch.setattr(utils_module, "_get_prisma_env", lambda: configured) + monkeypatch.setattr(utils_module.prisma_toolchain, "run_prisma", run_prisma) + + ProxyExtrasDBManager.apply_autorouter_daily_coverage() assert migrated["DATABASE_URL"] == expected assert migrated["PRISMA_OFFLINE_MODE"] == "true" assert configured["DATABASE_URL"] == pooled + run_prisma.assert_called_once() + assert run_prisma.call_args.kwargs["env"] == migrated class _MigrateDeployHarness: diff --git a/tests/proxy_behavior/spend/conftest.py b/tests/proxy_behavior/spend/conftest.py index 0b918401eac..450e2311613 100644 --- a/tests/proxy_behavior/spend/conftest.py +++ b/tests/proxy_behavior/spend/conftest.py @@ -1,12 +1,20 @@ """Session-scoped Prisma client for spend-rollup behavior tests against a real Postgres.""" +import asyncio +from collections.abc import AsyncIterator +from typing import Final + import pytest_asyncio +from litellm_proxy_extras.utils import ProxyExtrasDBManager from prisma import Prisma @pytest_asyncio.fixture(scope="session", loop_scope="session") -async def db(): - client = Prisma() +async def db() -> AsyncIterator[Prisma]: + await asyncio.to_thread(ProxyExtrasDBManager.apply_autorouter_daily_coverage) + client: Final = Prisma() await client.connect() - yield client - await client.disconnect() + try: + yield client + finally: + await client.disconnect() diff --git a/tests/proxy_behavior/spend/test_autorouter_daily_spend.py b/tests/proxy_behavior/spend/test_autorouter_daily_spend.py new file mode 100644 index 00000000000..0d65cb57339 --- /dev/null +++ b/tests/proxy_behavior/spend/test_autorouter_daily_spend.py @@ -0,0 +1,273 @@ +import json +import os +import time +import uuid +from collections.abc import Mapping +from datetime import timedelta +from typing import Final +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import pytest +from prisma import Prisma +from prisma.errors import RawQueryError + +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.db.autorouter_daily_spend import AUTOROUTER_DAILY_COSTS_SQL, AutoRouterDailyCosts +from litellm.proxy.db.autorouter_historical_spend import AUTOROUTER_HISTORICAL_COSTS_SQL, recover_daily_router_costs +from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, build_bulk_upsert, merge_by_conflict_key +from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING +from litellm.proxy.utils import PrismaClient, ProxyLogging + +pytestmark = pytest.mark.asyncio(loop_scope="session") + + +def _daily(day: str, key: str, user: str, llm: float, classifier: float, saved: float) -> Mapping[str, object]: + return { + "date": day, + "api_key": key, + "user_id": user, + "model": "model", + "custom_llm_provider": "provider", + "spend": llm, + "api_requests": 1, + "successful_requests": 1, + "autorouter_accounted_requests": 1, + "autorouter_requests": 1, + "autorouter_llm_spend": llm, + "autorouter_classifier_cost": classifier, + "autorouter_classifier_cost_recorded_requests": 1, + "autorouter_estimated_requests": 1, + "autorouter_estimated_actual_spend": llm + classifier, + "autorouter_savings_spend": saved, + } + + +async def test_daily_costs_use_request_dates_and_filters_instead_of_session_boundaries(db: Prisma) -> None: + async with db.tx() as tx: + await tx.execute_raw( + 'CREATE TEMP TABLE "LiteLLM_DailyUserSpend" (LIKE public."LiteLLM_DailyUserSpend" INCLUDING ALL) ON COMMIT DROP' + ) + rows: Final = ( + _daily("2026-09-21", "a", "owner-a", 8.0, 0.0, 4.0), + _daily("2026-09-22", "a", "owner-a", 1.8, 0.2, 1.0), + _daily("2026-09-22", "a", "owner-a", 3.0, 0.0, 2.0), + _daily("2026-09-22", "b", "owner-b", 7.0, 0.0, 5.0), + ) + table: Final = DAILY_SPEND_TABLES["user"] + sql, values = build_bulk_upsert(table, merge_by_conflict_key(table, rows)) + await tx.execute_raw(sql, *values) + for key, user, expected_spend, saved, requests in ( + (None, None, 12.0, 8.0, 3), + ("a", None, 5.0, 3.0, 2), + (None, "owner-a", 5.0, 3.0, 2), + ("a", "owner-b", 0.0, 0.0, 0), + ): + result: Final = await tx.query_raw(AUTOROUTER_DAILY_COSTS_SQL, "2026-09-22", "2026-09-22", key, user) + costs: Final = AutoRouterDailyCosts.model_validate(result[0]) + assert costs.complete and costs.requests == requests + assert costs.recorded_spend == pytest.approx(expected_spend) + assert costs.saved_spend == saved + assert costs.baseline_spend(saved) == pytest.approx(expected_spend + saved) + + +async def test_an_old_writer_cannot_certify_partial_daily_costs_as_complete(db: Prisma) -> None: + async with db.tx() as tx: + await tx.execute_raw( + 'CREATE TEMP TABLE "LiteLLM_DailyUserSpend" (LIKE public."LiteLLM_DailyUserSpend" INCLUDING ALL) ON COMMIT DROP' + ) + table: Final = DAILY_SPEND_TABLES["user"] + row: Final = _daily("2026-09-22", "a", "owner", 2.0, 0.1, 3.0) + sql, values = build_bulk_upsert(table, merge_by_conflict_key(table, (row,))) + await tx.execute_raw(sql, *values) + await tx.execute_raw( + 'UPDATE "LiteLLM_DailyUserSpend" SET api_requests=api_requests+1, successful_requests=successful_requests+1, ' + "spend=spend+8, autorouter_savings_spend=autorouter_savings_spend+4" + ) + result: Final = await tx.query_raw(AUTOROUTER_DAILY_COSTS_SQL, "2026-09-22", "2026-09-22", None, None) + costs: Final = AutoRouterDailyCosts.model_validate(result[0]) + assert costs.coverage == "partial" + assert costs.recorded_spend == pytest.approx(2.1) + assert costs.saved_spend == 7.0 + assert costs.baseline_spend(7.0) is None + + +async def test_retained_historical_costs_reconcile_through_prisma_and_reject_a_missing_free_request(db: Prisma) -> None: + async with db.tx() as tx: + await tx.execute_raw( + 'CREATE TEMP TABLE "LiteLLM_DailyUserSpend" (LIKE public."LiteLLM_DailyUserSpend" INCLUDING ALL) ON COMMIT DROP' + ) + await tx.execute_raw( + 'CREATE TEMP TABLE "LiteLLM_SpendLogs" (LIKE public."LiteLLM_SpendLogs" INCLUDING ALL) ON COMMIT DROP' + ) + cases: Final = ( + ("2026-09-21", "a", "owner-a", 90.0, 1.0, 9.0), + ("2026-09-22", "a", "owner-a", 2.0, 0.2, 3.0), + ("2026-09-22", "b", "owner-b", 7.0, 0.5, 5.0), + ) + rows: Final = tuple( + { + "date": day, + "api_key": key, + "user_id": user, + "model": "removed-historical-model", + "custom_llm_provider": "removed-provider", + "endpoint": "/responses", + "spend": llm + classifier, + "api_requests": 2, + "successful_requests": 2, + "prompt_tokens": 25, + "completion_tokens": 10, + "autorouter_savings_spend": saved, + } + for day, key, user, llm, classifier, saved in cases + ) + table: Final = DAILY_SPEND_TABLES["user"] + sql, values = build_bulk_upsert(table, merge_by_conflict_key(table, rows)) + await tx.execute_raw(sql, *values) + logs: Final = tuple( + { + "request_id": f"{day}-{key}-{suffix}", + "started_at": f"{day}T12:00:00", + "api_key": key, + "user_id": user, + "spend": spend, + "prompt_tokens": prompt, + "completion_tokens": completion, + "metadata": metadata, + } + for day, key, user, llm, classifier, saved in cases + for suffix, spend, prompt, completion, metadata in ( + ( + "routed", + llm, + 20, + 10, + { + "routing_decision": {"router_model_name": "retired-router", "classifier_cost": classifier}, + "autorouter_savings": saved, + }, + ), + ("free", 0.0, 0, 0, {}), + ( + "classifier", + classifier, + 5, + 0, + { + "internal_call_origin": "autorouter_classifier", + "routing_decision": {"router_model_name": "retired-router", "classifier_cost": classifier}, + }, + ), + ) + ) + await tx.execute_raw( + """INSERT INTO "LiteLLM_SpendLogs" + (request_id, "startTime", "endTime", call_type, api_key, "user", model, + custom_llm_provider, spend, prompt_tokens, completion_tokens, status, metadata) + SELECT request_id, started_at::timestamp, started_at::timestamp, 'aresponses', api_key, user_id, + 'removed-historical-model', 'removed-provider', spend, prompt_tokens, completion_tokens, + 'success', metadata + FROM jsonb_to_recordset($1::jsonb) AS rows( + request_id text, started_at text, api_key text, user_id text, + spend float8, prompt_tokens int, completion_tokens int, metadata jsonb + )""", + json.dumps(logs), + ) + for key, user, actual, saved, requests in ( + (None, None, 9.7, 8.0, 2), + ("a", None, 2.2, 3.0, 1), + (None, "owner-a", 2.2, 3.0, 1), + ("a", "owner-a", 2.2, 3.0, 1), + ("a", "owner-b", 0.0, 0.0, 0), + ): + result: Final = await tx.query_raw( + AUTOROUTER_HISTORICAL_COSTS_SQL, + "2026-09-22", + "2026-09-22", + key, + user, + json.dumps(ROUTE_ENDPOINT_MAPPING), + ) + recovered: Final = AutoRouterDailyCosts.model_validate(result[0]) + assert recovered.complete and recovered.comparison_complete + assert recovered.requests == requests + assert recovered.recorded_spend == pytest.approx(actual) + assert recovered.saved_spend == saved + assert recovered.baseline_spend(saved) == pytest.approx(actual + saved) + + await tx.execute_raw('DELETE FROM "LiteLLM_SpendLogs" WHERE request_id = $1', "2026-09-22-a-free") + incomplete: Final = await tx.query_raw( + AUTOROUTER_HISTORICAL_COSTS_SQL, + "2026-09-22", + "2026-09-22", + "a", + None, + json.dumps(ROUTE_ENDPOINT_MAPPING), + ) + costs: Final = AutoRouterDailyCosts.model_validate(incomplete[0]) + assert not costs.complete + assert costs.saved_spend == 3.0 + assert costs.recorded_spend is None + assert costs.baseline_spend(3.0) is None + + +async def test_blocked_historical_recovery_times_out_without_changing_costs_or_read_connection( + db: Prisma, monkeypatch: pytest.MonkeyPatch +) -> None: + identity: Final = uuid.uuid4().hex + database_url: Final = os.environ["DATABASE_URL"] + parsed: Final = urlsplit(database_url) + query: Final = urlencode({**dict(parse_qsl(parsed.query)), "connection_limit": "1"}) + monkeypatch.setenv("DATABASE_URL_READ_REPLICA", urlunsplit(parsed._replace(query=query))) + client: Final = PrismaClient(database_url, ProxyLogging(UserApiKeyCache())) + row: Final = { + **_daily("2026-09-22", identity, identity, 2.0, 0.1, 7.0), + "api_requests": 2, + "successful_requests": 2, + "spend": 10.0, + } + table: Final = DAILY_SPEND_TABLES["user"] + statement, values = build_bulk_upsert(table, merge_by_conflict_key(table, (row,))) + await db.execute_raw(statement, *values) + try: + await client.db.connect() + await client.read_db.execute_raw("SET statement_timeout = 4500") + connection_query: Final = ( + "SELECT pg_backend_pid() AS pid, current_setting('statement_timeout') AS timeout, " + "current_setting('transaction_read_only') AS read_only" + ) + connection_before: Final = await client.read_db.query_raw(connection_query) + daily_before: Final = await client.read_db.query_raw( + AUTOROUTER_DAILY_COSTS_SQL, + "2026-09-22", + "2026-09-22", + identity, + identity, + ) + recorded: Final = AutoRouterDailyCosts.model_validate(daily_before[0]) + assert recorded.recorded_spend == pytest.approx(2.1) and recorded.saved_spend == 7.0 + assert recorded.coverage == "partial" + async with db.tx(timeout=timedelta(seconds=8)) as blocker: + await blocker.execute_raw('LOCK TABLE "LiteLLM_SpendLogs" IN ACCESS EXCLUSIVE MODE') + started: Final = time.monotonic() + with pytest.raises(RawQueryError, match="canceling statement due to statement timeout"): + await recover_daily_router_costs(client, "2026-09-22", "2026-09-22", identity, identity) + assert time.monotonic() - started < 3.5 + assert await client.read_db.query_raw(connection_query) == connection_before + assert ( + await client.read_db.query_raw( + AUTOROUTER_DAILY_COSTS_SQL, + "2026-09-22", + "2026-09-22", + identity, + identity, + ) + == daily_before + ) + recovered: Final = await recover_daily_router_costs(client, "2026-09-22", "2026-09-22", identity, identity) + assert recovered is not None + assert recovered.recorded_spend == recorded.recorded_spend and recovered.saved_spend == recorded.saved_spend + assert await client.read_db.query_raw(connection_query) == connection_before + finally: + await client.db.disconnect() + await db.execute_raw('DELETE FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1', identity) diff --git a/tests/proxy_behavior/spend/test_baseline_accounting.py b/tests/proxy_behavior/spend/test_baseline_accounting.py index 3504751d132..b3b01cfbbea 100644 --- a/tests/proxy_behavior/spend/test_baseline_accounting.py +++ b/tests/proxy_behavior/spend/test_baseline_accounting.py @@ -15,10 +15,12 @@ from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction from litellm.proxy.db.baseline_accounting import ( BaselineAccountingRecord, BaselineAccountingStore, + BaselinePublication, DailyBaselineAttribution, DailyBaselineTarget, ) from litellm.proxy.db.create_views import SupportsRawQueries +from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, build_bulk_upsert, merge_by_conflict_key from litellm.proxy.spend_tracking.baseline_accounting import BaselineObservation from litellm.proxy.spend_tracking.savings import BaselineCostSnapshot from litellm.types.utils import Usage @@ -107,6 +109,11 @@ async def _user_sessions(db: Prisma, record: BaselineAccountingRecord) -> dict[s return {str(row["user_id"]): row for row in rows} +async def _daily_user(db: Prisma, record: BaselineAccountingRecord) -> dict[str, object]: + rows: Final = await db.query_raw('SELECT * FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1', record.api_key) + return rows[0] + + async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None: store: Final = _store(db) late: Final = record("late", 10001.0, user_id="late-user") @@ -117,6 +124,10 @@ async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma, before: Final = await _session(db, late) assert before["savings_estimated_actual_spend"] == before["spend"] == 0.17 assert before["saved_spend"] == 0.0 + before_daily: Final = await _daily_user(db, late) + assert before_daily["autorouter_estimated_requests"] == 1 + assert before_daily["autorouter_estimated_actual_spend"] == before["spend"] + assert before_daily["autorouter_savings_spend"] == 0.0 before_users: Final = await _user_sessions(db, late) assert set(before_users) == {"late-user"} assert before_users["late-user"]["savings_estimated_turns"] == 1 @@ -126,6 +137,8 @@ async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma, pending: Final = await _session(db, late) assert pending["spend"] == 0.34 and pending["savings_estimated_turns"] == 0 assert pending["saved_spend"] == pending["savings_estimated_actual_spend"] == 0.0 + pending_daily: Final = await _daily_user(db, late) + assert pending_daily["autorouter_estimated_requests"] == pending_daily["autorouter_estimated_actual_spend"] == 0 pending_users: Final = await _user_sessions(db, late) assert set(pending_users) == {"late-user", "early-user"} for user in pending_users.values(): @@ -143,6 +156,14 @@ async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma, assert logs[0]["spend"] == 0.17 assert logs[0]["metadata"]["autorouter_savings_estimate"]["provenance"] == "modeled" assert after["saved_spend"] == pytest.approx(logs[0]["metadata"]["autorouter_savings"]) + after_daily: Final = await _daily_user(db, late) + assert after_daily["autorouter_estimated_requests"] == after["savings_estimated_turns"] + assert after_daily["autorouter_estimated_actual_spend"] == after["savings_estimated_actual_spend"] + for field in ( + "api_requests", "autorouter_accounted_requests", "autorouter_requests", "autorouter_llm_spend", + "autorouter_classifier_cost", "autorouter_classifier_cost_recorded_requests", + ): + assert after_daily[field] == 0 after_users: Final = await _user_sessions(db, late) assert after_users["early-user"] == pending_users["early-user"] for field in ( @@ -179,6 +200,10 @@ async def test_commit_ack_loss_and_concurrent_duplicate_delivery_are_idempotent( session: Final = await _session(db, event) assert session["turns"] == session["savings_estimated_turns"] == 2 assert session["spend"] == session["savings_estimated_actual_spend"] == 0.34 + daily: Final = await _daily_user(db, event) + assert daily["autorouter_estimated_requests"] == session["savings_estimated_turns"] + assert daily["autorouter_estimated_actual_spend"] == session["savings_estimated_actual_spend"] + assert daily["autorouter_accounted_requests"] == daily["autorouter_requests"] == 0 users: Final = await _user_sessions(db, event) assert set(users) == ({"first-user", "second-user"} if attributed else set()) for user in users.values(): @@ -312,3 +337,155 @@ async def test_native_observation_enters_spend_pipeline_once_with_shared_daily_a assert tag_rows[0]["spend"] == tag_rows[0]["api_requests"] == 0 finally: await client.db.disconnect() + + +async def test_old_publisher_transitions_keep_daily_comparison_complete_after_retention( + db: Prisma, record: Callable[..., BaselineAccountingRecord] +) -> None: + event: Final = record("old-publisher") + assert event.daily is not None + assert await _store(db).append(event) == "recorded" + existing_actual: Final = 5.0 + other_savings: Final = 3.0 + actual: Final = event.pricing.actual_spend + row: Final = { + **event.daily.model_dump(exclude={"targets"}), + "user_id": event.api_key, + "spend": existing_actual + actual, + "api_requests": 2, + "successful_requests": 2, + "autorouter_accounted_requests": 2, + "autorouter_requests": 2, + "autorouter_llm_spend": existing_actual + actual, + "autorouter_classifier_cost_recorded_requests": 2, + "autorouter_estimated_requests": 1, + "autorouter_estimated_actual_spend": existing_actual, + "autorouter_savings_spend": other_savings, + } + table: Final = DAILY_SPEND_TABLES["user"] + statement, values = build_bulk_upsert(table, merge_by_conflict_key(table, (row,))) + await db.execute_raw(statement, *values) + + for status, baseline, savings_delta, expected_savings, count, covered_actual in ( + ("estimated", actual + 0.5, 0.5, other_savings + 0.5, 2, existing_actual + actual), + ("estimated", actual, -0.5, other_savings, 2, existing_actual + actual), + ("unknown", None, 0.0, other_savings, 1, existing_actual), + ("estimated", actual + 0.2, 0.2, other_savings + 0.2, 2, existing_actual + actual), + ("estimated", actual + 0.2, 0.0, other_savings + 0.2, 2, existing_actual + actual), + ): + publication: Final = BaselinePublication( + comparison_id=event.scope, + comparison_started_at=event.observation.started_at, + status=status, + reason="legacy-publisher-transition", + actual_spend=actual if baseline is not None else None, + baseline_spend=baseline, + ) + async with db.tx() as tx: + if savings_delta: + await tx.execute_raw( + 'UPDATE "LiteLLM_DailyUserSpend" SET autorouter_savings_spend=autorouter_savings_spend+$1::float8 ' + "WHERE api_key=$2 AND user_id=$3", + savings_delta, + event.api_key, + event.api_key, + ) + await tx.execute_raw( + 'UPDATE "LiteLLM_AutoRouterBaselineObservation" SET publication=$1 WHERE request_id=$2', + publication.model_dump_json(), + event.observation.request_id, + ) + daily: Final = await _daily_user(db, event) + assert daily["autorouter_estimated_requests"] == count + assert daily["autorouter_estimated_actual_spend"] == pytest.approx(covered_actual) + assert daily["autorouter_savings_spend"] == pytest.approx(expected_savings) + assert daily["api_requests"] == daily["autorouter_accounted_requests"] == daily["autorouter_requests"] == 2 + assert daily["spend"] == daily["autorouter_llm_spend"] == pytest.approx(existing_actual + actual) + assert daily["autorouter_classifier_cost"] == 0 + assert daily["autorouter_classifier_cost_recorded_requests"] == 2 + shadow: Final = await db.query_raw( + "SELECT publication::jsonb = daily_costs_publication::jsonb AS accounted " + 'FROM "LiteLLM_AutoRouterBaselineObservation" WHERE request_id=$1', + event.observation.request_id, + ) + assert shadow == [{"accounted": True}] + + retained: Final = await _daily_user(db, event) + await db.execute_raw( + 'DELETE FROM "LiteLLM_AutoRouterBaselineObservation" WHERE request_id=$1', event.observation.request_id + ) + assert await _daily_user(db, event) == retained + + +@pytest.mark.parametrize("targets", ("missing", "duplicate", "null-and-empty")) +async def test_legacy_zero_publication_repairs_shadow_once_for_normalized_user_targets( + db: Prisma, record: Callable[..., BaselineAccountingRecord], targets: str +) -> None: + source: Final = record("legacy-zero") + assert source.daily is not None + identities: Final = ( + () if targets == "missing" else ((None, "") if targets == "null-and-empty" else (source.api_key,) * 2) + ) + event: Final = source.model_copy( + update={ + "daily": source.daily.model_copy( + update={ + "targets": tuple(DailyBaselineTarget(entity="user", entity_id=identity) for identity in identities), + } + ) + } + ) + publication: Final = BaselinePublication( + comparison_id=event.scope, + comparison_started_at=event.observation.started_at, + status="estimated", + reason="legacy-zero", + actual_spend=event.pricing.actual_spend, + baseline_spend=event.pricing.actual_spend, + ).model_dump_json() + await db.execute_raw( + 'INSERT INTO "LiteLLM_AutoRouterBaselineObservation" (request_id,scope,started_at,revision,data,publication) ' + "VALUES ($1,$2,$3::float8,1,$4,$5)", + event.observation.request_id, + event.scope, + event.observation.started_at, + event.model_dump_json(), + publication, + ) + before: Final = await db.query_raw( + 'SELECT daily_costs_publication FROM "LiteLLM_AutoRouterBaselineObservation" WHERE request_id=$1', + event.observation.request_id, + ) + assert before == [{"daily_costs_publication": None}] + for _ in range(2): + await db.execute_raw( + 'UPDATE "LiteLLM_AutoRouterBaselineObservation" SET publication=$1 WHERE request_id=$2', + publication, + event.observation.request_id, + ) + rows: Final = await db.query_raw( + "SELECT user_id,autorouter_estimated_requests,autorouter_estimated_actual_spend,autorouter_savings_spend, " + 'api_requests,autorouter_requests,spend FROM "LiteLLM_DailyUserSpend" WHERE api_key=$1', + event.api_key, + ) + assert rows == ( + [ + { + "user_id": "" if targets == "null-and-empty" else event.api_key, + "autorouter_estimated_requests": 1, + "autorouter_estimated_actual_spend": event.pricing.actual_spend, + "autorouter_savings_spend": 0, + "api_requests": 0, + "autorouter_requests": 0, + "spend": 0, + } + ] + if identities + else [] + ) + shadow: Final = await db.query_raw( + "SELECT publication::jsonb = daily_costs_publication::jsonb AS accounted " + 'FROM "LiteLLM_AutoRouterBaselineObservation" WHERE request_id=$1', + event.observation.request_id, + ) + assert shadow == [{"accounted": True}] diff --git a/tests/proxy_migration_tests/test_autorouter_baseline_state.py b/tests/proxy_migration_tests/test_autorouter_baseline_state.py index d9021414bc4..1c0bc88fbf5 100644 --- a/tests/proxy_migration_tests/test_autorouter_baseline_state.py +++ b/tests/proxy_migration_tests/test_autorouter_baseline_state.py @@ -1,9 +1,11 @@ """Idempotent journal migration and primary transactional ownership.""" import asyncio +import json import os import time from collections.abc import AsyncGenerator, Iterator +from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager from datetime import timedelta from pathlib import Path @@ -101,3 +103,319 @@ async def test_locked_projection_is_bounded_and_cancellation_propagates( with pytest.raises(asyncio.CancelledError): await pending assert await store.project("scope") == "unchanged" + + +_COVERAGE_MIGRATION: Final = Path(__file__).parents[2] / ( + "litellm-proxy-extras/litellm_proxy_extras/migrations/20260923000000_add_daily_autorouter_costs/migration.sql" +) + + +@pytest.fixture +def _coverage_postgresql() -> Iterator[psycopg.Connection[tuple[object, ...]]]: + base: Final = os.environ["DATABASE_URL"].split("?")[0] + name: Final = f"coverage_{uuid4().hex}" + with psycopg.connect(base, autocommit=True) as admin: + admin.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(name))) + try: + with psycopg.connect(base, dbname=name) as connection: + yield connection + finally: + admin.execute(sql.SQL("DROP DATABASE {}").format(sql.Identifier(name))) + + +_COVERAGE_BASE: Final = """ +CREATE TABLE "LiteLLM_DailyUserSpend" ( + id TEXT PRIMARY KEY, user_id TEXT, date TEXT NOT NULL, api_key TEXT NOT NULL, + model TEXT, custom_llm_provider TEXT, mcp_namespaced_tool_name TEXT, endpoint TEXT, + model_group TEXT, spend DOUBLE PRECISION DEFAULT 0, autorouter_savings_spend DOUBLE PRECISION DEFAULT 0, + updated_at TIMESTAMP, + UNIQUE (user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint) +); +CREATE TABLE "LiteLLM_AutoRouterBaselineObservation" ( + request_id TEXT PRIMARY KEY, data TEXT, publication TEXT +); +INSERT INTO "LiteLLM_DailyUserSpend" + (id,user_id,date,api_key,model,custom_llm_provider,mcp_namespaced_tool_name,endpoint,spend,autorouter_savings_spend) + VALUES ('recorded','owner','2026-09-23','key','','','','',10,30) +""" + + +def _coverage_tables(conn: psycopg.Connection[tuple[object, ...]]) -> None: + conn.execute(_COVERAGE_BASE) + conn.execute( + 'INSERT INTO "LiteLLM_AutoRouterBaselineObservation" (request_id,data) VALUES (%s,%s)', + ( + "request", + json.dumps( + { + "daily": { + "date": "2026-09-23", + "api_key": "key", + "targets": [{"entity": "user", "entity_id": "owner"}], + } + } + ), + ), + ) + conn.commit() + + +def _coverage_install(conn: psycopg.Connection[tuple[object, ...]]) -> None: + conn.execute(_COVERAGE_MIGRATION.read_text()) + + +def _coverage_publish(conn: psycopg.Connection[tuple[object, ...]]) -> tuple[object, ...] | None: + conn.execute( + 'UPDATE "LiteLLM_AutoRouterBaselineObservation" SET publication=%s WHERE request_id=%s', + (json.dumps({"status": "estimated", "actual_spend": 2, "baseline_spend": 2}), "request"), + ) + row: Final = conn.execute( + "SELECT id,spend,autorouter_savings_spend,autorouter_estimated_requests,autorouter_estimated_actual_spend " + 'FROM "LiteLLM_DailyUserSpend"' + ).fetchone() + conn.commit() + return row + + +def _coverage_repeat_during_traffic(conn: psycopg.Connection[tuple[object, ...]]) -> None: + with ( + psycopg.connect(conn.info.dsn, password=conn.info.password) as traffic, + psycopg.connect(conn.info.dsn, password=conn.info.password, autocommit=True) as installer, + ): + traffic.execute('SELECT * FROM "LiteLLM_DailyUserSpend"') + traffic.execute('UPDATE "LiteLLM_AutoRouterBaselineObservation" SET data=data') + with ThreadPoolExecutor(max_workers=1) as pool: + repeated: Final = pool.submit(_coverage_install, installer) + try: + repeated.result(timeout=2) + finally: + installer.cancel() + traffic.rollback() + + +def test_coverage_install_preserves_history_and_repeat_does_not_wait_for_table_traffic( + _coverage_postgresql: psycopg.Connection[tuple[object, ...]], +) -> None: + _coverage_tables(_coverage_postgresql) + _coverage_install(_coverage_postgresql) + _coverage_repeat_during_traffic(_coverage_postgresql) + + assert _coverage_publish(_coverage_postgresql) == ("recorded", 10, 30, 1, 2) + assert _coverage_publish(_coverage_postgresql) == ("recorded", 10, 30, 1, 2) + + +@pytest.mark.parametrize("damage", ("missing", "disabled", "wrong-function", "wrong-event", "wrong-body")) +def test_coverage_installer_repairs_trigger_before_publishing_without_rewriting_history( + _coverage_postgresql: psycopg.Connection[tuple[object, ...]], + damage: str, +) -> None: + _coverage_tables(_coverage_postgresql) + _coverage_install(_coverage_postgresql) + if damage == "disabled": + _coverage_postgresql.execute( + 'ALTER TABLE "LiteLLM_AutoRouterBaselineObservation" ' + "DISABLE TRIGGER litellm_update_daily_autorouter_coverage" + ) + elif damage == "wrong-body": + _coverage_postgresql.execute( + "CREATE OR REPLACE FUNCTION litellm_update_daily_autorouter_coverage() " + "RETURNS TRIGGER LANGUAGE plpgsql AS 'BEGIN RETURN NEW; END;'" + ) + else: + _coverage_postgresql.execute( + 'DROP TRIGGER litellm_update_daily_autorouter_coverage ON "LiteLLM_AutoRouterBaselineObservation"' + ) + if damage == "wrong-function": + _coverage_postgresql.execute( + "CREATE FUNCTION coverage_noop() RETURNS TRIGGER LANGUAGE plpgsql AS 'BEGIN RETURN NEW; END;'" + ) + _coverage_postgresql.execute( + "CREATE TRIGGER litellm_update_daily_autorouter_coverage BEFORE UPDATE OF publication " + 'ON "LiteLLM_AutoRouterBaselineObservation" FOR EACH ROW EXECUTE FUNCTION coverage_noop()' + ) + elif damage == "wrong-event": + _coverage_postgresql.execute( + "CREATE TRIGGER litellm_update_daily_autorouter_coverage BEFORE UPDATE OF data " + 'ON "LiteLLM_AutoRouterBaselineObservation" FOR EACH ROW ' + "EXECUTE FUNCTION litellm_update_daily_autorouter_coverage()" + ) + _coverage_postgresql.commit() + + _coverage_install(_coverage_postgresql) + + assert _coverage_publish(_coverage_postgresql) == ("recorded", 10, 30, 1, 2) + + +@pytest.mark.parametrize("version", (None, 2)) +def test_coverage_installer_retains_compatible_unversioned_and_newer_functions( + _coverage_postgresql: psycopg.Connection[tuple[object, ...]], + version: int | None, +) -> None: + _coverage_tables(_coverage_postgresql) + _coverage_install(_coverage_postgresql) + before: Final = _coverage_postgresql.execute( + "SELECT prosrc FROM pg_proc WHERE oid='litellm_update_daily_autorouter_coverage()'::regprocedure" + ).fetchone() + assert before is not None and isinstance(before[0], str) + body: Final = before[0] if version is None else "\n" + before[0] + _coverage_postgresql.execute( + sql.SQL( + "CREATE OR REPLACE FUNCTION litellm_update_daily_autorouter_coverage() " + "RETURNS TRIGGER LANGUAGE plpgsql AS {}" + ).format(sql.Literal(body)) + ) + _coverage_postgresql.execute( + sql.SQL("COMMENT ON FUNCTION litellm_update_daily_autorouter_coverage() IS {}").format( + sql.Literal(None if version is None else f"litellm:autorouter_daily_coverage:{version}") + ) + ) + _coverage_postgresql.commit() + + _coverage_repeat_during_traffic(_coverage_postgresql) + + after: Final = _coverage_postgresql.execute( + "SELECT prosrc,obj_description(oid,'pg_proc') FROM pg_proc " + "WHERE oid='litellm_update_daily_autorouter_coverage()'::regprocedure" + ).fetchone() + assert after == (body, None if version is None else f"litellm:autorouter_daily_coverage:{version}") + assert _coverage_publish(_coverage_postgresql) == ("recorded", 10, 30, 1, 2) + + +@pytest.mark.parametrize( + "incompatible,reason", + ( + ( + 'ALTER TABLE "LiteLLM_DailyUserSpend" ALTER COLUMN autorouter_estimated_requests TYPE INTEGER', + "incompatible column definitions", + ), + ( + 'ALTER TABLE "LiteLLM_DailyUserSpend" ALTER COLUMN autorouter_estimated_requests SET DEFAULT 99', + "incompatible column definitions", + ), + ( + "COMMENT ON FUNCTION litellm_update_daily_autorouter_coverage() IS 'litellm:autorouter_daily_coverage:2';" + "ALTER FUNCTION litellm_update_daily_autorouter_coverage() IMMUTABLE", + "Newer daily auto-router coverage is incompatible", + ), + ( + "COMMENT ON FUNCTION litellm_update_daily_autorouter_coverage() IS 'litellm:autorouter_daily_coverage:2';" + 'ALTER TABLE "LiteLLM_DailyUserSpend" DROP COLUMN autorouter_estimated_actual_spend', + "Newer daily auto-router coverage is incompatible", + ), + pytest.param( + "COMMENT ON FUNCTION litellm_update_daily_autorouter_coverage() IS 'litellm:autorouter_daily_coverage:2';" + 'DROP TRIGGER litellm_update_daily_autorouter_coverage ON "LiteLLM_AutoRouterBaselineObservation";' + "CREATE TRIGGER litellm_update_daily_autorouter_coverage BEFORE UPDATE OF data " + 'ON "LiteLLM_AutoRouterBaselineObservation" FOR EACH ROW ' + "EXECUTE FUNCTION litellm_update_daily_autorouter_coverage()", + "Newer daily auto-router coverage is incompatible", + id="newer-trigger-event", + ), + pytest.param( + "COMMENT ON FUNCTION litellm_update_daily_autorouter_coverage() IS 'litellm:autorouter_daily_coverage:2';" + 'ALTER TABLE "LiteLLM_AutoRouterBaselineObservation" ' + "DISABLE TRIGGER litellm_update_daily_autorouter_coverage", + "Newer daily auto-router coverage is incompatible", + id="newer-trigger-disabled", + ), + pytest.param( + "COMMENT ON FUNCTION litellm_update_daily_autorouter_coverage() IS 'litellm:autorouter_daily_coverage:2';" + 'DROP TRIGGER litellm_update_daily_autorouter_coverage ON "LiteLLM_AutoRouterBaselineObservation"', + "Newer daily auto-router coverage is incompatible", + id="newer-trigger-missing", + ), + ), +) +def test_coverage_installer_rejects_incompatible_state_without_replacing_it( + _coverage_postgresql: psycopg.Connection[tuple[object, ...]], + incompatible: str, + reason: str, +) -> None: + _coverage_tables(_coverage_postgresql) + _coverage_install(_coverage_postgresql) + _coverage_postgresql.execute(incompatible) + state_query: Final = ( + "SELECT p.prosrc,p.provolatile,obj_description(p.oid,'pg_proc'),t.oid,pg_get_triggerdef(t.oid),t.tgenabled " + "FROM pg_proc p LEFT JOIN pg_trigger t " + "ON t.tgrelid='\"LiteLLM_AutoRouterBaselineObservation\"'::regclass " + "AND t.tgname='litellm_update_daily_autorouter_coverage' " + "WHERE p.oid='litellm_update_daily_autorouter_coverage()'::regprocedure" + ) + before: Final = _coverage_postgresql.execute(state_query).fetchone() + assert before is not None + _coverage_postgresql.commit() + + with pytest.raises(psycopg.errors.RaiseException, match=reason): + _coverage_install(_coverage_postgresql) + + _coverage_postgresql.rollback() + assert _coverage_postgresql.execute(state_query).fetchone() == before + assert _coverage_postgresql.execute( + 'SELECT id,spend,autorouter_savings_spend FROM "LiteLLM_DailyUserSpend"' + ).fetchall() == [("recorded", 10, 30)] + + +def test_coverage_installer_does_not_trust_or_modify_public_schema_homonyms( + _coverage_postgresql: psycopg.Connection[tuple[object, ...]], +) -> None: + _coverage_tables(_coverage_postgresql) + _coverage_install(_coverage_postgresql) + _coverage_postgresql.execute('CREATE SCHEMA "tenant coverage"') + _coverage_postgresql.execute('SET search_path TO "tenant coverage", public') + _coverage_tables(_coverage_postgresql) + + _coverage_install(_coverage_postgresql) + + assert _coverage_publish(_coverage_postgresql) == ("recorded", 10, 30, 1, 2) + assert _coverage_postgresql.execute( + 'SELECT spend,autorouter_savings_spend,autorouter_estimated_requests FROM public."LiteLLM_DailyUserSpend"' + ).fetchall() == [(10, 30, 0)] + assert _coverage_postgresql.execute( + 'SELECT daily_costs_publication FROM public."LiteLLM_AutoRouterBaselineObservation"' + ).fetchall() == [(None,)] + + +def _coverage_waiting(conn: psycopg.Connection[tuple[object, ...]], pid: int, kind: str) -> None: + deadline: Final = time.monotonic() + 3 + while time.monotonic() < deadline: + if conn.execute( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=%s AND locktype=%s AND NOT granted)", (pid, kind) + ).fetchone() == (True,): + return + time.sleep(0.01) + pytest.fail(f"Installer did not reach its expected {kind} wait") + + +def test_waiting_coverage_installer_rechecks_after_peer_finishes_before_taking_table_locks( + _coverage_postgresql: psycopg.Connection[tuple[object, ...]], +) -> None: + _coverage_tables(_coverage_postgresql) + _coverage_postgresql.autocommit = True + info: Final = _coverage_postgresql.info + with ( + psycopg.connect(info.dsn, password=info.password) as original_reader, + psycopg.connect(info.dsn, password=info.password) as next_reader, + psycopg.connect(info.dsn, password=info.password, autocommit=True) as first, + psycopg.connect(info.dsn, password=info.password, autocommit=True) as second, + ThreadPoolExecutor(max_workers=3) as pool, + ): + original_reader.execute('SELECT * FROM "LiteLLM_DailyUserSpend"') + installing: Final = pool.submit(_coverage_install, first) + try: + _coverage_waiting(_coverage_postgresql, first.info.backend_pid, "relation") + queued_read: Final = pool.submit(next_reader.execute, 'SELECT * FROM "LiteLLM_DailyUserSpend"') + _coverage_waiting(_coverage_postgresql, next_reader.info.backend_pid, "relation") + waiting: Final = pool.submit(_coverage_install, second) + _coverage_waiting(_coverage_postgresql, second.info.backend_pid, "advisory") + original_reader.rollback() + installing.result(timeout=2) + queued_read.result(timeout=2) + waiting.result(timeout=2) + finally: + first.cancel() + second.cancel() + next_reader.cancel() + original_reader.rollback() + next_reader.rollback() + + assert _coverage_publish(_coverage_postgresql) == ("recorded", 10, 30, 1, 2) diff --git a/tests/test_litellm/proxy/client/cli/test_statusline_script.py b/tests/test_litellm/proxy/client/cli/test_statusline_script.py index 0cbeec86ee8..6e4c3500ffe 100644 --- a/tests/test_litellm/proxy/client/cli/test_statusline_script.py +++ b/tests/test_litellm/proxy/client/cli/test_statusline_script.py @@ -1,6 +1,7 @@ """The status line script is copied verbatim to the user's machine, so these drive it the way Claude Code and Codex do: the documented stdin payload, a transcript on disk, and the proxy behind an injected fetch.""" +import hashlib import io import json import os @@ -155,6 +156,23 @@ class TestCredentials: class TestSessionCache: + def test_old_coverage_only_cache_is_refreshed_before_displaying_totals(self, tmp_path: Path) -> None: + credentials: Final = Credentials("http://p", "sk") + identity: Final = "\n".join((credentials.base_url, credentials.api_key, SESSION_ID)) + old_path: Final = tmp_path / hashlib.sha256(identity.encode()).hexdigest() + old_path.write_text(json.dumps({ + "fetched_at": 100.0, + "session": { + **RECORDED._asdict(), "spend": 10.0, "baseline_spend": 1.5, + "turns": 3, "savings_estimated_turns": 1, "savings_estimated_actual_spend": 2.0, + }, + })) + + def fetch(credentials: Credentials, session_id: str) -> Fetched: + return Fetched(RECORDED, True) + + assert load_session(credentials, SESSION_ID, tmp_path, fetch, now=lambda: 101.0) == RECORDED + def test_a_definite_answer_is_served_from_the_cache_within_the_ttl(self, tmp_path): calls = [] @@ -342,6 +360,47 @@ class TestRender: class TestClaudeCodeMode: + @pytest.mark.parametrize("field", ("baseline_spend", "savings_estimated_baseline_spend")) + @pytest.mark.parametrize("invalid_baseline", ("1.5", True, float("nan"), float("inf"))) + def test_invalid_baselines_do_not_display_a_cost_comparison( + self, config_dir: Path, field: str, invalid_baseline: object, + ) -> None: + session: Final = statusline_script._session_from_payload({ + **RECORDED._asdict(), field: invalid_baseline, + }) + assert session is None + assert render("m", session, config_dir, use_color=False) == "Routed to: m" + + @pytest.mark.parametrize("estimated_turns, saved_spend, delta", ( + (0, 30.0, "-75%"), (0, -2.0, "+25%"), + (1, 30.0, "-75%"), (1, 0.0, "0%"), (1, -2.0, "+25%"), + )) + def test_historical_and_current_savings_remain_combined_after_cache_refresh( + self, tmp_path: Path, transcript: Path, config_dir: Path, estimated_turns: int, + saved_spend: float, delta: str, + ) -> None: + session: Final = statusline_script._session_from_payload( + { + **RECORDED._asdict(), + "spend": 10.0, + "baseline_spend": 10.0 + saved_spend, + "savings_estimated_baseline_spend": 1.5 if estimated_turns else None, + "turns": 3, + "savings_estimated_turns": estimated_turns, + "savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0, + } + ) + assert session is not None + + def fetch(credentials: Credentials, session_id: str) -> Fetched: + return Fetched(session, True) + + first: Final = _run(_payload(transcript), _env(tmp_path, config_dir), fetch) + assert first == _run(_payload(transcript), _env(tmp_path, config_dir), fetch) + assert first.splitlines()[0] == f"Routed to: claude-sonnet-5 {delta} vs Claude Opus 5" + assert "$10.00" in first and f"${10.0 + saved_spend:.2f}" in first + assert "$1.50" not in first and "Savings unavailable" not in first + @pytest.mark.parametrize("estimated_turns", (0, 1)) def test_current_estimates_keep_the_routed_model_and_compare_only_covered_turns( self, tmp_path: Path, transcript: Path, config_dir: Path, estimated_turns: int diff --git a/tests/test_litellm/proxy/db/conftest.py b/tests/test_litellm/proxy/db/conftest.py index 4d775142dd5..084240e300c 100644 --- a/tests/test_litellm/proxy/db/conftest.py +++ b/tests/test_litellm/proxy/db/conftest.py @@ -100,6 +100,15 @@ if not earlier_calls and os.environ.get("FAKE_PRISMA_HANG_FIRST"): grandchild = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)"]) pathlib.Path(os.environ["FAKE_PRISMA_GRANDCHILD_PIDFILE"]).write_text(str(grandchild.pid)) time.sleep(600) +if sys.argv[1:3] == ["db", "execute"]: + if os.environ.get("FAKE_PRISMA_FAIL_DB_EXECUTE"): + sys.exit(1) + if not any(json.loads(call)[:2] == ["db", "execute"] for call in earlier_calls): + first_failure = os.environ.get("FAKE_PRISMA_FAIL_FIRST_DB_EXECUTE") + if first_failure == "timeout": + time.sleep(600) + if first_failure: + sys.exit(1) sys.exit(0) """ @@ -142,6 +151,8 @@ def fake_prisma_cli(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Generato monkeypatch.setenv("FAKE_PRISMA_GRANDCHILD_PIDFILE", str(cli.grandchild_pidfile)) monkeypatch.setenv("LITELLM_PRISMA_COMMAND_TIMEOUT", "1") monkeypatch.delenv("FAKE_PRISMA_HANG_FIRST", raising=False) + monkeypatch.delenv("FAKE_PRISMA_FAIL_DB_EXECUTE", raising=False) + monkeypatch.delenv("FAKE_PRISMA_FAIL_FIRST_DB_EXECUTE", raising=False) yield cli if cli.grandchild_pidfile.exists(): try: diff --git a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py index c17ba75db03..c4fb85cea1b 100644 --- a/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py +++ b/tests/test_litellm/proxy/db/db_transaction_queue/test_daily_spend_update_queue.py @@ -209,6 +209,13 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key(): "prompt_caching_savings_spend": 0, "gateway_injected_caching_savings_spend": 0, "autorouter_savings_spend": 0, + "autorouter_accounted_requests": 0, + "autorouter_requests": 0, + "autorouter_llm_spend": 0, + "autorouter_classifier_cost": 0, + "autorouter_classifier_cost_recorded_requests": 0, + "autorouter_estimated_requests": 0, + "autorouter_estimated_actual_spend": 0, "total_response_time_ms": 0, "timed_requests": 0, } @@ -263,6 +270,13 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions( "prompt_caching_savings_spend": 0, "gateway_injected_caching_savings_spend": 0, "autorouter_savings_spend": 0, + "autorouter_accounted_requests": 0, + "autorouter_requests": 0, + "autorouter_llm_spend": 0, + "autorouter_classifier_cost": 0, + "autorouter_classifier_cost_recorded_requests": 0, + "autorouter_estimated_requests": 0, + "autorouter_estimated_actual_spend": 0, "total_response_time_ms": 0, "timed_requests": 0, } @@ -549,7 +563,7 @@ async def test_every_optional_daily_metric_aggregates(daily_spend_update_queue): # additive metrics may be declared NotRequired[float] for rows queued by a pod # running the previous release, so unwrap before matching args = get_args(annotation) - return (args[0] if args else annotation) in (int, float) + return _numeric(args[0]) if args else annotation in (int, float) numeric_fields = [ name for name, annotation in BaseDailySpendTransaction.__annotations__.items() if _numeric(annotation) @@ -592,3 +606,22 @@ async def test_optional_metric_missing_from_an_older_payload_still_aggregates( assert updates[0][test_key]["autorouter_savings_spend"] == pytest.approx(0.25) assert updates[0][test_key]["total_response_time_ms"] == 900 assert updates[0][test_key]["timed_requests"] == 1 + + +@pytest.mark.parametrize("reverse", (False, True)) +def test_old_daily_payloads_do_not_gain_router_accounting_coverage(reverse: bool) -> None: + old = {"spend": 1.0, "prompt_tokens": 2, "completion_tokens": 1, + "api_requests": 1, "successful_requests": 1, "failed_requests": 0} + current = {**old, "autorouter_accounted_requests": 1, "autorouter_requests": 1, + "autorouter_llm_spend": 1.0, "autorouter_classifier_cost": 0.2, + "autorouter_classifier_cost_recorded_requests": 1, + "autorouter_estimated_requests": 1, "autorouter_estimated_actual_spend": 1.2} + updates = [{"key": current}, {"key": old}] if reverse else [{"key": old}, {"key": current}] + merged = DailySpendUpdateQueue.get_aggregated_daily_spend_update_transactions(updates)["key"] + assert merged["api_requests"] == 2 + assert merged["autorouter_accounted_requests"] == merged["autorouter_requests"] == 1 + assert merged["autorouter_llm_spend"] == 1.0 + assert merged["autorouter_classifier_cost"] == 0.2 + assert merged["autorouter_classifier_cost_recorded_requests"] == merged["autorouter_estimated_requests"] == 1 + assert merged["autorouter_estimated_actual_spend"] == 1.2 + assert "autorouter_accounted_requests" not in old diff --git a/tests/test_litellm/proxy/db/test_autorouter_historical_spend.py b/tests/test_litellm/proxy/db/test_autorouter_historical_spend.py new file mode 100644 index 00000000000..eae367abb90 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_autorouter_historical_spend.py @@ -0,0 +1,286 @@ +import json +import re +from collections.abc import Mapping +from typing import Final + +import psycopg +import pytest +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from pytest_postgresql import factories + +from litellm.proxy.db.autorouter_daily_spend import AutoRouterDailyCosts +from litellm.proxy.db.autorouter_historical_spend import AUTOROUTER_HISTORICAL_COSTS_SQL +from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING +from litellm.proxy.spend_tracking.savings import classifier_cost_from_decision, known_autorouter_savings + +_historical_postgresql_proc: Final = factories.postgresql_proc() +_historical_postgresql: Final = factories.postgresql("_historical_postgresql_proc") +_DDL: Final = """ +CREATE TABLE "LiteLLM_DailyUserSpend" ( + date TEXT, user_id TEXT, api_key TEXT, model TEXT, custom_llm_provider TEXT, + mcp_namespaced_tool_name TEXT, endpoint TEXT, + api_requests BIGINT DEFAULT 0, successful_requests BIGINT DEFAULT 0, failed_requests BIGINT DEFAULT 0, + prompt_tokens BIGINT DEFAULT 0, completion_tokens BIGINT DEFAULT 0, spend FLOAT8 DEFAULT 0, + autorouter_requests BIGINT DEFAULT 0, autorouter_llm_spend FLOAT8 DEFAULT 0, + autorouter_classifier_cost FLOAT8 DEFAULT 0, autorouter_classifier_cost_recorded_requests BIGINT DEFAULT 0, + autorouter_estimated_requests BIGINT DEFAULT 0, autorouter_estimated_actual_spend FLOAT8 DEFAULT 0, + autorouter_savings_spend FLOAT8 DEFAULT 0, autorouter_accounted_requests BIGINT DEFAULT 0 +); +CREATE TABLE "LiteLLM_SpendLogs" ( + request_id TEXT PRIMARY KEY, "startTime" TIMESTAMP, "user" TEXT, api_key TEXT, + model TEXT, custom_llm_provider TEXT, mcp_namespaced_tool_name TEXT, call_type TEXT, + spend FLOAT8, prompt_tokens BIGINT, completion_tokens BIGINT, status TEXT, metadata JSONB +) +""" + + +def _seed(conn: psycopg.Connection) -> None: + conn.execute(_DDL) + conn.execute( + """INSERT INTO "LiteLLM_DailyUserSpend" + (date, user_id, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint, + api_requests, successful_requests, prompt_tokens, completion_tokens, spend, autorouter_savings_spend) + VALUES ('2026-09-22', 'owner', 'key', 'model', NULL, '', '/chat/completions', 2, 2, 55, 10, 5.1, 4)""" + ) + decision: Final = {"router_model_name": "router", "classifier_cost": 0.1} + rows: Final = ( + ("routed", 2.0, 20, 10, {"routing_decision": decision, "autorouter_savings": 4}), + ("ordinary", 3.0, 30, 0, {}), + ("classifier", 0.1, 5, 0, {"internal_call_origin": "autorouter_classifier", "routing_decision": decision}), + ) + for request_id, spend, prompt, completion, metadata in rows: + conn.execute( + """INSERT INTO "LiteLLM_SpendLogs" + (request_id, "startTime", "user", api_key, model, custom_llm_provider, mcp_namespaced_tool_name, + call_type, spend, prompt_tokens, completion_tokens, status, metadata) + VALUES (%s, '2026-09-22 12:00:00', 'owner', 'key', 'model', '', NULL, + 'acompletion', %s, %s, %s, 'success', %s)""", + (request_id, spend, prompt, completion, Jsonb(metadata)), + ) + + +def _read(conn: psycopg.Connection, *, key: str | None = None, user: str | None = None) -> AutoRouterDailyCosts: + query: Final = re.sub(r"\$(\d+)", r"%(p\1)s", AUTOROUTER_HISTORICAL_COSTS_SQL) + with conn.cursor(row_factory=dict_row) as cursor: + cursor.execute( + query, # pyright: ignore[reportArgumentType] # fixed production SQL uses Prisma parameter syntax + {"p1": "2026-09-22", "p2": "2026-09-22", "p3": key, "p4": user, "p5": json.dumps(ROUTE_ENDPOINT_MAPPING)}, + ) + return AutoRouterDailyCosts.model_validate(cursor.fetchone()) + + +def test_historical_costs_include_no_session_turns_and_classifier_charge_once( + _historical_postgresql: psycopg.Connection, +) -> None: + _seed(_historical_postgresql) + + costs: Final = _read(_historical_postgresql, key="key", user="owner") + + assert costs == AutoRouterDailyCosts( + requests=1, + llm_spend=2, + classifier_cost=0.1, + classifier_requests=1, + estimated_requests=1, + estimated_actual_spend=2.1, + saved_spend=4, + ) + assert costs.recorded_spend == pytest.approx(2.1) + assert costs.baseline_spend(4) == pytest.approx(6.1) + assert _read(_historical_postgresql, key="other").requests == 0 + assert _read(_historical_postgresql, user="other").requests == 0 + + +@pytest.mark.parametrize("missing_request", ("ordinary", "classifier", "routed")) +def test_missing_logs_cannot_certify_costs_even_when_saved_savings_match( + _historical_postgresql: psycopg.Connection, missing_request: str +) -> None: + _seed(_historical_postgresql) + _historical_postgresql.execute('DELETE FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (missing_request,)) + + costs: Final = _read(_historical_postgresql) + + assert not costs.complete + assert costs.saved_spend == 4 + assert costs.recorded_spend is None + assert costs.baseline_spend(4) is None + + +def test_missing_zero_cost_request_invalidates_matching_spend_and_token_totals( + _historical_postgresql: psycopg.Connection, +) -> None: + _seed(_historical_postgresql) + _historical_postgresql.execute( + """UPDATE "LiteLLM_DailyUserSpend" SET spend = 2.1, prompt_tokens = 25; + DELETE FROM "LiteLLM_SpendLogs" WHERE request_id = 'ordinary'""" + ) + + costs: Final = _read(_historical_postgresql) + + assert not costs.complete + assert costs.recorded_spend is None + assert costs.saved_spend == 4 + + +@pytest.mark.parametrize("changed", ("model", "custom_llm_provider", "mcp_namespaced_tool_name", "call_type")) +def test_equal_totals_in_different_daily_identity_do_not_certify_recovery( + _historical_postgresql: psycopg.Connection, changed: str +) -> None: + _seed(_historical_postgresql) + _historical_postgresql.execute( + psycopg.sql.SQL('UPDATE "LiteLLM_SpendLogs" SET {} = %s WHERE request_id = %s').format( + psycopg.sql.Identifier(changed) + ), + ("different", "ordinary"), + ) + + costs: Final = _read(_historical_postgresql) + + assert not costs.complete + assert costs.recorded_spend is None + + +@pytest.mark.parametrize( + "metadata,saved,estimated,comparison", + ( + ({"autorouter_savings": 0}, 0, 1, True), + ({"autorouter_savings": -1}, -1, 1, True), + ({"autorouter_savings": 0, "autorouter_savings_estimate": {"version": 3, "status": "unknown"}}, 0, 0, True), + ({"autorouter_savings": 0, "autorouter_savings_estimate": {"version": 3, "status": "estimated"}}, 0, 1, True), + ({}, 4, 0, False), + ({"autorouter_savings": True}, 4, 0, False), + ({"autorouter_savings": "4"}, 4, 0, False), + ({"autorouter_savings": 4, "autorouter_savings_estimate": {}}, 4, 0, False), + ( + {"autorouter_savings": 4, "autorouter_savings_estimate": {"version": 1.0, "status": "estimated"}}, + 4, + 0, + False, + ), + ( + {"autorouter_savings": 4, "autorouter_savings_estimate": {"version": True, "status": "estimated"}}, + 4, + 0, + False, + ), + ({"autorouter_savings": 4, "autorouter_savings_estimate": {"version": 99, "status": "estimated"}}, 4, 0, False), + ), +) +def test_recorded_estimate_provenance_gates_baseline_without_hiding_actual_cost( + _historical_postgresql: psycopg.Connection, + metadata: Mapping[str, object], + saved: float, + estimated: int, + comparison: bool, +) -> None: + _seed(_historical_postgresql) + _historical_postgresql.execute( + 'UPDATE "LiteLLM_SpendLogs" SET metadata = %s WHERE request_id = %s', + (Jsonb({"routing_decision": {"router_model_name": "router", "classifier_cost": 0.1}, **metadata}), "routed"), + ) + _historical_postgresql.execute('UPDATE "LiteLLM_DailyUserSpend" SET autorouter_savings_spend = %s', (saved,)) + + costs: Final = _read(_historical_postgresql) + estimate: Final = metadata.get("autorouter_savings_estimate") + recorded: Final = known_autorouter_savings( + model=None, + custom_llm_provider=None, + routing_decision=None, + usage_object=None, + recorded_autorouter_savings=metadata.get("autorouter_savings"), + recorded_autorouter_savings_estimate=estimate if isinstance(estimate, Mapping) else None, + ) + + assert costs.complete and costs.recorded_spend == pytest.approx(2.1) + assert costs.estimated_requests == estimated == int(recorded is not None) + assert costs.comparison_complete is comparison + assert costs.saved_spend == saved + assert costs.baseline_spend(saved) == (pytest.approx(2.1 + saved) if estimated and comparison else None) + + +def test_unknown_classifier_preserves_recorded_subtotal_with_partial_coverage( + _historical_postgresql: psycopg.Connection, +) -> None: + _seed(_historical_postgresql) + _historical_postgresql.execute( + """UPDATE "LiteLLM_SpendLogs" SET metadata = metadata #- '{routing_decision,classifier_cost}' + WHERE request_id = 'routed'""" + ) + + costs: Final = _read(_historical_postgresql) + + assert costs.complete and costs.recorded_llm_spend == 2 + assert costs.recorded_classifier_cost is None + assert costs.recorded_spend == 2 + assert costs.coverage == "partial" + assert costs.baseline_spend(4) is None + + +@pytest.mark.parametrize("classifier,expected", ((-0.1, -0.1), (0, 0), (True, None), ("0.1", None))) +def test_classifier_numeric_contract_matches_daily_owner( + _historical_postgresql: psycopg.Connection, classifier: object, expected: float | None +) -> None: + _seed(_historical_postgresql) + _historical_postgresql.execute( + """UPDATE "LiteLLM_SpendLogs" SET metadata = jsonb_set(metadata, '{routing_decision,classifier_cost}', %s) + WHERE request_id = 'routed'""", + (Jsonb(classifier),), + ) + + costs: Final = _read(_historical_postgresql) + + assert costs.recorded_classifier_cost == expected == classifier_cost_from_decision({"classifier_cost": classifier}) + assert costs.classifier_requests == int(expected is not None) + assert costs.recorded_spend == pytest.approx(2 + (expected or 0)) + + +def test_unknown_routed_request_is_counted_in_actual_but_excluded_from_baseline( + _historical_postgresql: psycopg.Connection, +) -> None: + _seed(_historical_postgresql) + _historical_postgresql.execute( + 'UPDATE "LiteLLM_SpendLogs" SET metadata = %s WHERE request_id = %s', + ( + Jsonb( + { + "routing_decision": {"router_model_name": "router", "classifier_cost": 0}, + "autorouter_savings_estimate": {"version": 3, "status": "unknown"}, + } + ), + "ordinary", + ), + ) + + costs: Final = _read(_historical_postgresql) + + assert costs.complete and costs.comparison_complete + assert costs.requests == 2 and costs.estimated_requests == 1 + assert costs.recorded_spend == pytest.approx(5.1) + assert costs.baseline_spend(4) == pytest.approx(6.1) + + +def test_historical_logs_replace_partial_bucket_and_retain_durable_other_days( + _historical_postgresql: psycopg.Connection, +) -> None: + _seed(_historical_postgresql) + _historical_postgresql.execute( + """UPDATE "LiteLLM_DailyUserSpend" SET autorouter_accounted_requests = 1, + autorouter_requests = 1, autorouter_llm_spend = 1, autorouter_classifier_cost = 0.05, + autorouter_classifier_cost_recorded_requests = 1, autorouter_estimated_requests = 1, + autorouter_estimated_actual_spend = 1.05; + INSERT INTO "LiteLLM_DailyUserSpend" + (date, user_id, api_key, model, endpoint, api_requests, successful_requests, + autorouter_accounted_requests, autorouter_requests, autorouter_llm_spend, + autorouter_classifier_cost, autorouter_classifier_cost_recorded_requests, + autorouter_estimated_requests, autorouter_estimated_actual_spend, autorouter_savings_spend) + VALUES ('2026-09-22', 'owner', 'key', 'durable-model', '/chat/completions', 1, 1, 1, 1, 3, 0.2, 1, 1, 3.2, 2), + ('2026-09-21', 'owner', 'key', 'durable-model', '/chat/completions', 1, 1, 1, 1, 90, 1, 1, 1, 91, 9)""" + ) + + costs: Final = _read(_historical_postgresql) + + assert costs.complete and costs.requests == 2 + assert costs.recorded_spend == pytest.approx(5.3) + assert costs.saved_spend == 6 + assert costs.baseline_spend(6) == pytest.approx(11.3) diff --git a/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py b/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py index 7893fb82281..3a5e72e3929 100644 --- a/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py +++ b/tests/test_litellm/proxy/db/test_daily_spend_bulk_upsert.py @@ -3,6 +3,7 @@ import re from collections.abc import AsyncIterator from contextlib import AbstractAsyncContextManager, asynccontextmanager +from typing import Final import pytest @@ -55,20 +56,30 @@ def test_conflict_key_normalizes_every_nullable_key_column(column): @pytest.mark.parametrize("order", [("null_first"), ("empty_first")]) -def test_null_and_empty_provider_merge_into_one_row(order): +@pytest.mark.parametrize("entity", ["user", "tag"]) +def test_null_and_empty_provider_merge_into_one_row(order, entity): """Two queue entries differing only in NULL versus '' arbitrate to the same row. Postgres rejects one statement touching a row twice, so they must be folded first. Asserted under both input orders: a single ordering would prove nothing here.""" - null_entry = tag_txn(custom_llm_provider=None, spend=0.25, api_requests=1) - empty_entry = tag_txn(custom_llm_provider="", spend=0.75, api_requests=3) + metrics: Final = { + "autorouter_accounted_requests": 1, "autorouter_requests": 1, + "autorouter_llm_spend": 0.5, "autorouter_classifier_cost": 0.125, + "autorouter_classifier_cost_recorded_requests": 1, + "autorouter_estimated_requests": 1, "autorouter_estimated_actual_spend": 0.625, + } + null_entry = tag_txn(custom_llm_provider=None, spend=0.25, api_requests=1, **metrics) + empty_entry = tag_txn(custom_llm_provider="", spend=0.75, api_requests=3, **metrics) transactions = (null_entry, empty_entry) if order == "null_first" else (empty_entry, null_entry) - merged = merge_by_conflict_key(TAG_TABLE, transactions) + merged = merge_by_conflict_key(DAILY_SPEND_TABLES[entity], transactions) assert len(merged) == 1 _, folded = merged[0] assert folded["spend"] == pytest.approx(1.0) assert folded["api_requests"] == 4 + if entity == "user": + for field, value in metrics.items(): + assert folded[field] == value * 2 def test_distinct_keys_are_not_merged_and_are_ordered_deterministically(): diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py index daa07c8224a..b3a9f3c8b6a 100644 --- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py +++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py @@ -2789,6 +2789,7 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent( payload = { "request_id": "req-no-compression", + "status": "success", "user": "test-user", "startTime": "2026-07-17T00:00:00", "api_key": "test-key", @@ -2819,6 +2820,8 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent( assert transaction["prompt_caching_savings_spend"] == 0 assert transaction["spend"] == 0.01 assert transaction["autorouter_savings_spend"] == expected + assert transaction["autorouter_estimated_requests"] == int(expected != 0.0) + assert transaction["autorouter_estimated_actual_spend"] == (0.015 if expected != 0.0 else 0.0) # --------------------------------------------------------------------------- @@ -3580,6 +3583,104 @@ async def test_daily_transaction_internal_call_keeps_spend_but_not_request_count assert user_sent["successful_requests"] == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize("saved, estimate", [ + (0.0, None), (-0.25, None), (0.5, None), + (0.0, {"version": 3, "status": "estimated"}), + (0.5, {"version": 3, "status": "unknown"}), +]) +@pytest.mark.parametrize("kind", ("routed", "plain", "failed", "internal", "malformed", "empty")) +async def test_daily_router_costs_account_for_every_external_request_without_a_session( + saved: float, estimate: dict[str, object] | None, kind: str, +) -> None: + writer: Final = DBSpendUpdateWriter() + prisma: Final = MagicMock() + prisma.get_request_status.return_value = "failure" if kind == "failed" else "success" + decision: Final = {"router_model_name": "auto", "classifier_cost": 0.125} + metadata: Final = { + "routing_decision": {"plain": None, "malformed": ["not-a-decision"], "empty": {}}.get(kind, decision), + "internal_call_origin": "autorouter_classifier" if kind == "internal" else None, + "autorouter_savings": saved, + "autorouter_savings_estimate": estimate, + } + transaction: Final = await writer._common_add_spend_log_transaction_to_daily_transaction( + payload={ + "user": "request-user", "startTime": "2026-09-23T23:59:59+00:00", "api_key": "hash", + "model": "selected", "model_group": "auto", "custom_llm_provider": "openai", + "call_type": "acompletion", "prompt_tokens": 10, "completion_tokens": 5, + "status": "failure" if kind == "failed" else "success", "spend": 0.5, "metadata": json.dumps(metadata), + }, + prisma_client=prisma, type="user", + ) + assert transaction is not None + routed: Final = kind == "routed" + estimated: Final = routed and (estimate is None or estimate["status"] == "estimated") + assert transaction["autorouter_accounted_requests"] == transaction["api_requests"] == int(kind != "internal") + assert transaction["autorouter_requests"] == int(routed) + assert transaction["autorouter_llm_spend"] == (0.5 if routed else 0.0) + assert transaction["autorouter_classifier_cost"] == (0.125 if routed else 0.0) + assert transaction["autorouter_classifier_cost_recorded_requests"] == int(routed) + assert transaction["autorouter_estimated_requests"] == int(estimated) + assert transaction["autorouter_estimated_actual_spend"] == (0.625 if estimated else 0.0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", ("success", None, "failure")) +async def test_daily_router_costs_use_status_normalized_from_request_metadata(status: str | None) -> None: + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + from litellm.proxy.utils import PrismaClient + + started_at: Final = datetime(2026, 9, 23, tzinfo=timezone.utc) + metadata: Final = { + "user_api_key": "test-hash", "user_api_key_user_id": "u", "model_group": "auto", + "routing_decision": {"router_model_name": "auto", "classifier_cost": 0.125}, + **({"status": status} if status is not None else {}), + } + payload: Final = get_logging_payload( + kwargs={ + "model": "synthetic-target", "call_type": "acompletion", "response_cost": 0.5, + "litellm_params": {"metadata": metadata, "custom_llm_provider": "openai"}, + }, + response_obj={}, start_time=started_at, end_time=started_at, + ) + prisma: Final = PrismaClient.__new__(PrismaClient) + transaction: Final = await DBSpendUpdateWriter()._common_add_spend_log_transaction_to_daily_transaction( + payload, prisma, "user", + ) + + assert transaction is not None + assert payload["status"] == ("failure" if status == "failure" else "success") + assert transaction["autorouter_requests"] == int(status != "failure") + assert transaction["autorouter_llm_spend"] == (0.0 if status == "failure" else 0.5) + assert transaction["autorouter_classifier_cost"] == (0.0 if status == "failure" else 0.125) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("classifier_cost, recorded", [(None, 0), (True, 0), (float("inf"), 0), (0.0, 1)]) +async def test_daily_router_classifier_zero_is_known_only_when_recorded( + classifier_cost: object, recorded: int, +) -> None: + prisma: Final = MagicMock() + prisma.get_request_status.return_value = "success" + transaction: Final = await DBSpendUpdateWriter()._common_add_spend_log_transaction_to_daily_transaction( + payload={ + "user": "u", "startTime": "2026-09-23T00:00:00", "api_key": "hash", "model": "m", + "model_group": "auto", "custom_llm_provider": "openai", "call_type": "acompletion", + "prompt_tokens": 1, "completion_tokens": 1, "spend": 1.0, "status": "success", + "metadata": json.dumps({ + "routing_decision": {"router_model_name": "auto", "classifier_cost": classifier_cost}, + "autorouter_savings": 0.0, + }), + }, prisma_client=prisma, type="user", + ) + assert transaction is not None + assert transaction["autorouter_requests"] == 1 + assert transaction["autorouter_classifier_cost_recorded_requests"] == recorded + assert transaction["autorouter_classifier_cost"] == 0.0 + assert transaction["autorouter_estimated_requests"] == 1 + assert transaction["autorouter_estimated_actual_spend"] == 1.0 + + def _response_time_payload(request_duration_ms: object, metadata: dict | None = None) -> dict: return { "request_id": "req-timed-1", diff --git a/tests/test_litellm/proxy/db/test_prisma_client.py b/tests/test_litellm/proxy/db/test_prisma_client.py index 99e494fccd5..487a9564d5f 100644 --- a/tests/test_litellm/proxy/db/test_prisma_client.py +++ b/tests/test_litellm/proxy/db/test_prisma_client.py @@ -3,6 +3,7 @@ import os import signal import sys import urllib.parse +from typing import TYPE_CHECKING from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -12,6 +13,9 @@ from fastapi.testclient import TestClient from litellm.proxy.db.prisma_client import PrismaManager, PrismaWrapper, should_update_prisma_schema +if TYPE_CHECKING: + from tests.test_litellm.proxy.db.conftest import FakePrismaCli + @pytest.fixture(autouse=True) def mock_prisma_binary(): @@ -213,7 +217,10 @@ def test_db_push_applies_replica_identity_full_when_requested(monkeypatch, fake_ assert PrismaManager.setup_database(use_migrate=False) is True - assert fake_prisma_cli.calls == [DB_PUSH_ARGV] + assert len(fake_prisma_cli.calls) == 2 + assert fake_prisma_cli.calls[0] == DB_PUSH_ARGV + assert fake_prisma_cli.calls[1][:2] == ["db", "execute"] + assert fake_prisma_cli.calls[1][-1].endswith("20260923000000_add_daily_autorouter_costs/migration.sql") assert applied == [True] @@ -246,7 +253,47 @@ def test_db_push_proceeds_when_spend_logs_is_not_partitioned(monkeypatch, fake_p ) assert PrismaManager.setup_database(use_migrate=False) is True - assert fake_prisma_cli.calls == [DB_PUSH_ARGV] + assert len(fake_prisma_cli.calls) == 2 + assert fake_prisma_cli.calls[0] == DB_PUSH_ARGV + assert fake_prisma_cli.calls[1][:2] == ["db", "execute"] + + +@pytest.mark.parametrize("use_migrate", [False, True]) +def test_db_push_cannot_succeed_without_daily_coverage_owner( + fake_prisma_cli: "FakePrismaCli", + unset_database_url: None, + monkeypatch: pytest.MonkeyPatch, + use_migrate: bool, +) -> None: + monkeypatch.setenv("FAKE_PRISMA_FAIL_DB_EXECUTE", "1") + + with patch("litellm_proxy_extras.utils.time.sleep"), pytest.raises(RuntimeError, match="daily auto-router coverage"): + PrismaManager.setup_database(use_migrate=use_migrate) + + assert [call[:2] for call in fake_prisma_cli.calls if call != ["--version"]] == [ + ["migrate", "deploy"] if use_migrate else ["db", "push"], + *([["db", "execute"]] * 4), + ] + + +@pytest.mark.parametrize("use_migrate", [False, True]) +@pytest.mark.parametrize("failure", ["exit", "timeout"]) +def test_database_setup_retries_transient_daily_coverage_install( + fake_prisma_cli: "FakePrismaCli", + unset_database_url: None, + monkeypatch: pytest.MonkeyPatch, + use_migrate: bool, + failure: str, +) -> None: + monkeypatch.setenv("FAKE_PRISMA_FAIL_FIRST_DB_EXECUTE", failure) + + with patch("litellm_proxy_extras.utils.time.sleep"): + assert PrismaManager.setup_database(use_migrate=use_migrate) is True + + assert [call[:2] for call in fake_prisma_cli.calls if call != ["--version"]] == [ + ["migrate", "deploy"] if use_migrate else ["db", "push"], + ["db", "execute"], ["db", "execute"], + ] def _entra_jwt(expires_in_seconds: int) -> str: @@ -439,7 +486,9 @@ def test_db_push_timeout_takes_its_process_tree_with_it(fake_prisma_cli, unset_d monkeypatch.setenv("FAKE_PRISMA_HANG_FIRST", "1") assert PrismaManager.setup_database(use_migrate=False) is True - assert fake_prisma_cli.calls == [DB_PUSH_ARGV, DB_PUSH_ARGV] + assert fake_prisma_cli.calls[:2] == [DB_PUSH_ARGV, DB_PUSH_ARGV] + assert len(fake_prisma_cli.calls) == 3 + assert fake_prisma_cli.calls[2][:2] == ["db", "execute"] assert fake_prisma_cli.grandchild_is_gone(within_seconds=5) diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index ff3d19e8637..eb4ba238617 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -2,7 +2,9 @@ Unit tests for auto router management endpoints """ +import json from collections.abc import Mapping, Sequence +from datetime import date from functools import partial from pathlib import Path from types import SimpleNamespace @@ -25,6 +27,8 @@ from litellm.proxy._types import ( ProxyException, UserAPIKeyAuth, ) +from litellm.proxy.db.autorouter_daily_spend import AUTOROUTER_DAILY_COSTS_SQL, AutoRouterDailyCosts +from litellm.proxy.db.autorouter_historical_spend import AUTOROUTER_HISTORICAL_COSTS_SQL from litellm.proxy.management_endpoints import auto_router_endpoints from litellm.proxy.management_endpoints.auto_router_endpoints import ( preview_auto_router_routing, @@ -40,6 +44,12 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( AutoRouterBenchmarksResponse, AutoRouterRoutingTestRequest, ) +from litellm.types.proxy.management_endpoints.common_daily_activity import ( + DailySpendData, + DailySpendMetadata, + SpendAnalyticsPaginatedResponse, + SpendMetrics, +) from litellm.types.router import Deployment from litellm.types.utils import Choices, Message, ModelResponse @@ -629,18 +639,39 @@ class TestAutoRouterBenchmarks: monkeypatch.setattr(proxy_server, "llm_router", None) + @pytest.fixture(autouse=True) + def _daily_savings(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints import common_daily_activity + + monkeypatch.setattr( + common_daily_activity, + "get_daily_activity_aggregated", + AsyncMock(return_value=self._daily_result(30.0)), + ) + + @staticmethod + def _daily_result(saved: float, *, has_rows: bool = True) -> SpendAnalyticsPaginatedResponse: + return SpendAnalyticsPaginatedResponse( + results=[DailySpendData(date=date(2026, 7, 15), metrics=SpendMetrics(autorouter_savings_spend=saved))] + if has_rows else [], + metadata=DailySpendMetadata(total_autorouter_savings_spend=saved), + ) + @staticmethod async def _benchmarks( monkeypatch: pytest.MonkeyPatch, rows: Sequence[Mapping[str, object]], model_list: Sequence[object], api_key: str | None = None, + costs: AutoRouterDailyCosts | None = None, ) -> AutoRouterBenchmarksResponse: from litellm.proxy import proxy_server from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks class _DB: async def query_raw(self, sql: str, *params: object): + if sql in (AUTOROUTER_DAILY_COSTS_SQL, AUTOROUTER_HISTORICAL_COSTS_SQL): + return [(costs or AutoRouterDailyCosts(complete=False)).model_dump()] return rows monkeypatch.setattr(proxy_server, "prisma_client", type("P", (), {"db": _DB()})()) @@ -719,24 +750,208 @@ class TestAutoRouterBenchmarks: assert totals.saved_pct == -100.0 assert totals.classifier_cost == 0.4 - @pytest.mark.parametrize("estimated_turns", [0, 4]) - def test_savings_compare_only_the_current_estimated_cohort(self, estimated_turns: int) -> None: - from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals - + @pytest.mark.asyncio + @pytest.mark.parametrize("estimated_turns,saved_spend,baseline_spend,saved_pct", [ + (0, 30.0, 40.0, 75.0), (4, 30.0, 40.0, 75.0), + (0, 0.0, None, None), (4, 0.0, 10.0, 0.0), (0, -2.0, 8.0, -25.0), + ]) + async def test_session_groups_preserve_history_and_only_compare_known_savings( + self, monkeypatch: pytest.MonkeyPatch, estimated_turns: int, saved_spend: float, + baseline_spend: float | None, saved_pct: float | None, + ) -> None: row: Final = self.ROW.model_copy( update={ + "saved_spend": saved_spend, "savings_estimated_turns": estimated_turns, "savings_estimated_actual_spend": 2.0 if estimated_turns else 0.0, "savings_estimated_saved_spend": -0.5 if estimated_turns else 0.0, } ) - totals: Final = _benchmark_totals(row) - assert totals.spend == 10.0 - assert totals.savings_estimated_turns == estimated_turns - assert totals.saved_spend == (-0.5 if estimated_turns else None) - assert totals.baseline_spend == (1.5 if estimated_turns else None) - assert totals.saved_pct == (pytest.approx(-33.3) if estimated_turns else None) - assert totals.saved_per_session is None + response: Final = await self._benchmarks(monkeypatch, rows=[row.model_dump()], model_list=[]) + group: Final = response.groups[0] + assert group.spend == 10.0 + assert group.saved_spend == saved_spend + assert group.baseline_spend == baseline_spend + assert group.saved_pct == saved_pct + assert group.saved_per_session == saved_spend / row.sessions + + @pytest.mark.asyncio + async def test_tracked_session_groups_keep_historical_and_new_estimates_once( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + historical: Final = self.ROW.model_copy( + update={ + "savings_estimated_turns": 0, + "savings_estimated_actual_spend": 0.0, + "savings_estimated_saved_spend": 0.0, + } + ) + current: Final = self.ROW.model_copy( + update={ + "router_name": "current-auto", + "sessions": 1, + "turns": 4, + "spend": 2.0, + "saved_spend": -0.5, + "savings_estimated_turns": 4, + "savings_estimated_actual_spend": 2.0, + "savings_estimated_saved_spend": -0.5, + } + ) + response: Final = await self._benchmarks( + monkeypatch, rows=[historical.model_dump(), current.model_dump()], model_list=[] + ) + assert [group.saved_spend for group in response.groups] == [30.0, -0.5] + assert response.groups[1].savings_estimated_turns == 4 + + @pytest.mark.asyncio + @pytest.mark.parametrize("api_key,user_id", [(None, None), ("key-hash", None), (None, "selected-user"), + ("key-hash", "selected-user")]) + @pytest.mark.parametrize( + "daily_saved,has_daily_rows,has_sessions", + [(6.5, True, True), (-0.5, True, True), (0.0, True, True), (30.0, True, True), + (2.0, True, False), (0.0, False, True), (0.0, False, False)], + ids=["historical-and-new-plus-no-session", "signed-loss", "recorded-zero", "matching-is-not-proof", + "no-session-history", "out-of-window-session-only", "empty"], + ) + async def test_total_uses_overall_request_date_savings_with_identical_filters( + self, monkeypatch: pytest.MonkeyPatch, api_key: str | None, user_id: str | None, + daily_saved: float, has_daily_rows: bool, has_sessions: bool, + ) -> None: + from litellm.proxy.management_endpoints import common_daily_activity + from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks + + daily: Final = AsyncMock(return_value=self._daily_result(daily_saved, has_rows=has_daily_rows)) + query: Final = AsyncMock(side_effect=[ + [self.ROW.model_dump()] if has_sessions else [], + [AutoRouterDailyCosts(complete=not has_daily_rows, saved_spend=daily_saved).model_dump()], + [AutoRouterDailyCosts(complete=not has_daily_rows, saved_spend=daily_saved).model_dump()], + ]) + prisma: Final = SimpleNamespace(db=SimpleNamespace(query_raw=query)) + monkeypatch.setattr(common_daily_activity, "get_daily_activity_aggregated", daily) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + response: Final = await get_auto_router_benchmarks( + user_api_key_dict=ADMIN, start_date="2026-07-01", end_date="2026-08-01", + api_key=api_key, user_id=user_id, + ) + + daily.assert_awaited_once_with( + prisma_client=prisma, table_name="litellm_dailyuserspend", entity_id_field="user_id", + entity_id=user_id, entity_metadata_field=None, start_date="2026-07-01", end_date="2026-08-01", + model=None, api_key=api_key, + ) + assert query.call_args_list[0].args[1:] == ("2026-07-01T00:00:00", "2026-08-02T00:00:00", api_key, user_id) + assert query.call_args_list[1].args[1:] == ("2026-07-01", "2026-08-01", api_key, user_id) + assert response.totals.saved_spend == daily_saved + assert response.totals.sessions == (4 if has_sessions else 0) + assert response.totals.saved_per_session == (7.5 if has_sessions else 0.0) + assert [group.saved_spend for group in response.groups] == ([30.0] if has_sessions else []) + assert ( + response.totals.spend, response.totals.classifier_cost, response.totals.baseline_spend, + response.totals.saved_pct, + ) == ((None,) * 4 if has_daily_rows else (0.0,) * 4) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "complete, classifier_requests, estimated, saved, recorded_saved, expected_spend, baseline, pct, coverage", + [ + (True, 3, 2, 30.0, 30.0, 10.0, 32.5, 92.3, "complete"), + (True, 3, 0, 0.0, 0.0, 10.0, None, None, "complete"), + (True, 2, 2, 30.0, 30.0, 10.0, None, None, "partial"), + (False, 3, 2, 30.0, 30.0, 10.0, None, None, "partial"), + (True, 3, 2, 30.0, 29.0, 10.0, None, None, "complete"), + (True, 3, 2, -0.5, -0.5, 10.0, 2.0, -25.0, "complete"), + ], + ids=["covered-subset", "unknown-only", "missing-classifier", "partial-history", "pending-savings", "loss"], + ) + async def test_daily_costs_preserve_actual_charges_and_compare_only_matching_estimates( + self, monkeypatch: pytest.MonkeyPatch, complete: bool, classifier_requests: int, + estimated: int, saved: float, recorded_saved: float, expected_spend: float | None, + baseline: float | None, pct: float | None, coverage: str, + ) -> None: + from litellm.proxy.management_endpoints import common_daily_activity + + monkeypatch.setattr(common_daily_activity, "get_daily_activity_aggregated", + AsyncMock(return_value=self._daily_result(saved))) + costs: Final = AutoRouterDailyCosts( + requests=3, llm_spend=9.7, classifier_cost=0.3, classifier_requests=classifier_requests, + estimated_requests=estimated, estimated_actual_spend=2.5 if estimated else 0.0, + saved_spend=recorded_saved, complete=complete, + ) + response: Final = await self._benchmarks(monkeypatch, rows=[], model_list=[], costs=costs) + totals: Final = response.totals + assert totals.saved_spend == saved + assert totals.spend == expected_spend + assert totals.llm_spend == 9.7 + assert totals.classifier_cost == (0.3 if classifier_requests == 3 else None) + assert totals.baseline_spend == baseline + assert totals.saved_pct == pct + assert totals.cost_coverage == coverage + assert totals.cost_requests == (3 if complete else None) + assert totals.savings_estimated_turns == estimated + assert totals.savings_estimated_actual_spend == (2.5 if estimated else 0.0) + assert totals.sessions == totals.turns == 0 + + @pytest.mark.asyncio + @pytest.mark.parametrize("recovery", ["complete", "partial", "empty", "failure", "invalid", "not-needed"]) + async def test_historical_cost_recovery_preserves_durable_savings_and_request_scope( + self, monkeypatch: pytest.MonkeyPatch, recovery: str, + ) -> None: + from litellm.proxy.management_endpoints import common_daily_activity + from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING + + recovered: Final = AutoRouterDailyCosts( + requests=3, llm_spend=4.5, classifier_cost=0.5, classifier_requests=3, + estimated_requests=2, estimated_actual_spend=3.0, saved_spend=6.5, + complete=recovery != "partial", + ) + recorded: Final = recovered if recovery == "not-needed" else AutoRouterDailyCosts( + requests=1, llm_spend=2.0, classifier_cost=0.25, classifier_requests=1, + estimated_requests=1, estimated_actual_spend=2.25, saved_spend=1.5, complete=False, + ) + recovery_result: Final = { + "empty": [], "failure": RuntimeError("retained logs unavailable"), "invalid": [{"unexpected": True}], + }.get(recovery, [recovered.model_dump()]) + query: Final = AsyncMock(side_effect=[[self.ROW.model_dump()], [recorded.model_dump()], recovery_result]) + transaction: Final = SimpleNamespace(query_raw=query, execute_raw=AsyncMock()) + reader: Final = MagicMock() + reader.tx.return_value.__aenter__.return_value = transaction + monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace( + db=SimpleNamespace(query_raw=query), read_db=reader, + )) + monkeypatch.setattr(common_daily_activity, "get_daily_activity_aggregated", + AsyncMock(return_value=self._daily_result(6.5))) + response: Final = await auto_router_endpoints.get_auto_router_benchmarks( + user_api_key_dict=ADMIN, start_date="2026-07-01", end_date="2026-08-01", + api_key="selected-key", user_id="selected-user", + ) + totals: Final = response.totals + assert totals.saved_spend == 6.5 + assert response.groups[0].saved_spend == self.ROW.saved_spend + fallback: Final = recovery in ("empty", "failure", "invalid") + assert (totals.spend, totals.llm_spend, totals.classifier_cost) == ( + (2.25, 2.0, 0.25) if fallback else (5.0, 4.5, 0.5) + ) + comparable: Final = recovery in ("complete", "not-needed") + assert (totals.baseline_spend, totals.saved_pct) == ((9.5, 68.4) if comparable else (None, None)) + assert totals.cost_coverage == ("complete" if comparable else "partial") + assert query.await_count == (2 if recovery == "not-needed" else 3) + if recovery != "not-needed": + sql, start, end, key, user, endpoints = query.call_args.args + assert sql == AUTOROUTER_HISTORICAL_COSTS_SQL + assert (start, end, key, user) == ("2026-07-01", "2026-08-01", "selected-key", "selected-user") + assert json.loads(endpoints) == ROUTE_ENDPOINT_MAPPING + + @pytest.mark.asyncio + async def test_daily_savings_failure_never_falls_back_to_session_savings(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.management_endpoints import common_daily_activity + + monkeypatch.setattr( + common_daily_activity, "get_daily_activity_aggregated", + AsyncMock(side_effect=HTTPException(status_code=500, detail="Daily savings unavailable")), + ) + with pytest.raises(HTTPException, match="Daily savings unavailable"): + await self._benchmarks(monkeypatch, rows=[self.ROW.model_dump()], model_list=[]) def test_an_empty_window_folds_to_zeros(self): from litellm.proxy.management_endpoints.auto_router_endpoints import ( @@ -854,6 +1069,8 @@ class TestAutoRouterBenchmarks: class _DB: async def query_raw(self, sql: str, *params: object): + if sql in (AUTOROUTER_DAILY_COSTS_SQL, AUTOROUTER_HISTORICAL_COSTS_SQL): + return [AutoRouterDailyCosts(complete=False).model_dump()] captured["sql"] = sql captured["params"] = params return [TestAutoRouterBenchmarks.ROW.model_dump()] @@ -870,9 +1087,12 @@ class TestAutoRouterBenchmarks: assert captured["params"] == ("2026-07-01T00:00:00", "2026-08-02T00:00:00", "key-hash", user_id) assert response.routers_in_scope == 1 assert response.groups[0].router_name == "live-auto" - assert response.groups[0].saved_pct == response.totals.saved_pct == 75.0 - assert response.groups[0].classifier_cost == response.totals.classifier_cost == 0.4 - assert response.totals.spend - response.totals.classifier_cost == pytest.approx(9.6) + assert response.groups[0].saved_pct == 75.0 + assert response.groups[0].classifier_cost == 0.4 + assert response.totals.saved_spend == 30.0 + assert response.totals.saved_pct is None + assert response.totals.classifier_cost is None + assert response.totals.spend is None @pytest.mark.asyncio @pytest.mark.parametrize("recorded_turns", [0, 3, 10]) @@ -899,12 +1119,12 @@ class TestAutoRouterBenchmarks: wire: Final = response.model_dump() assert wire["groups"][0]["classifier_cost"] == 0.4 assert wire["groups"][1]["classifier_cost"] == (pytest.approx(0.2) if recorded_turns == 10 else None) - assert wire["totals"]["classifier_cost"] == (pytest.approx(0.6) if recorded_turns == 10 else None) + assert wire["totals"]["classifier_cost"] is None assert response.totals.turns == 50 - assert response.totals.spend == 12.0 - assert response.totals.saved_spend == 29.5 - assert response.totals.baseline_spend == 41.5 - assert response.totals.saved_pct == 71.1 + assert response.totals.spend is None + assert response.totals.saved_spend == 30.0 + assert response.totals.baseline_spend is None + assert response.totals.saved_pct is None assert response.totals.saved_per_session == 5.9 @pytest.mark.asyncio @@ -919,6 +1139,8 @@ class TestAutoRouterBenchmarks: class _DB: async def query_raw(self, sql: str, *params: object): + if sql in (AUTOROUTER_DAILY_COSTS_SQL, AUTOROUTER_HISTORICAL_COSTS_SQL): + return [AutoRouterDailyCosts(complete=False).model_dump()] return [{**TestAutoRouterBenchmarks.ROW.model_dump(), "tier_turns": wire_value}] monkeypatch.setattr(proxy_server, "prisma_client", type("P", (), {"db": _DB()})()) @@ -953,7 +1175,7 @@ class TestAutoRouterBenchmarks: assert len(response.groups) == 3 assert response.routers_in_scope == 3 assert by_name["live-auto"].spend == 10.0 - assert response.totals.spend == 10.0 + assert response.totals.spend is None assert response.totals.sessions == 4 for name in ("idle-from-config", "idle-from-db"): idle = by_name[name] @@ -1100,13 +1322,17 @@ class TestAutoRouterSession: @pytest.mark.asyncio @pytest.mark.parametrize( - "turns, estimated", [(3, True), (10, True), (10, False)], ids=["full", "partial", "legacy"] + "turns, estimated, saved_spend", + [(3, True, 0.24), (10, True, 0.24), (10, False, 0.24), (10, False, 0.0), + (10, True, -0.24), (10, False, -0.24), (3, True, 0.0)], + ids=["full", "partial", "legacy", "unknown", "negative", "legacy-negative", "estimated-zero"], ) async def test_a_key_reads_its_own_session_with_the_baseline_its_turns_were_priced_against( self, monkeypatch: pytest.MonkeyPatch, turns: int, estimated: bool, + saved_spend: float, ) -> None: from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session @@ -1115,10 +1341,12 @@ class TestAutoRouterSession: key: value for key, value in self.ROW.items() if estimated or not key.startswith("savings_estimated_") } spend: Final = 0.14 if turns == 3 else 10.0 - if estimated and turns != 3: - row["savings_estimated_saved_spend"] = -0.04 + if estimated: + row["savings_estimated_saved_spend"] = saved_spend if turns == 3 else -0.04 self._rig( - monkeypatch, [{**row, "api_key": caller.api_key, "session_id": "sess-1", "turns": turns, "spend": spend}] + monkeypatch, + [{**row, "api_key": caller.api_key, "session_id": "sess-1", "turns": turns, "spend": spend, + "saved_spend": saved_spend}], ) response = await get_auto_router_session(user_api_key_dict=caller, session_id="sess-1") assert response.model_dump() == { @@ -1128,15 +1356,42 @@ class TestAutoRouterSession: "turns": turns, "last_model": "anthropic/claude-sonnet-5", "spend": spend, - "saved_spend": (0.24 if turns == 3 else -0.04) if estimated else None, + "saved_spend": saved_spend, "savings_estimated_turns": 3 if estimated else 0, "savings_estimated_actual_spend": 0.14 if estimated else 0.0, - "baseline_spend": pytest.approx(0.38) if turns == 3 else None, - "savings_estimated_baseline_spend": pytest.approx(0.38 if turns == 3 else 0.1) if estimated else None, - "baseline_model": "anthropic/claude-opus-5" if estimated else None, - "baseline_models": {"anthropic/claude-opus-5": 3} if estimated else {}, + "baseline_spend": pytest.approx(spend + saved_spend) if estimated or saved_spend != 0 else None, + "savings_estimated_baseline_spend": pytest.approx(spend + saved_spend if turns == 3 else 0.1) + if estimated else None, + "baseline_model": "anthropic/claude-opus-5", + "baseline_models": {"anthropic/claude-opus-5": 3}, } + @pytest.mark.asyncio + async def test_unknown_only_session_keeps_savings_unavailable_in_the_cli( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + ) -> None: + from litellm.proxy.client.cli.commands.statusline_script import _session_from_payload, render + from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session + + caller: Final = UserAPIKeyAuth(api_key="sk-caller") + self._rig(monkeypatch, [{ + **self.ROW, "api_key": caller.api_key, "session_id": "unknown-only", + "saved_spend": 0.0, "savings_estimated_turns": 0, + "savings_estimated_actual_spend": 0.0, "savings_estimated_saved_spend": 0.0, + "savings_estimated_baseline_models": {}, + }]) + + response: Final = await get_auto_router_session(user_api_key_dict=caller, session_id="unknown-only") + assert response.turns == 3 and response.savings_estimated_turns == 0 + assert response.baseline_model == "anthropic/claude-opus-5" + assert response.baseline_spend is None + assert response.savings_estimated_baseline_spend is None + session: Final = _session_from_payload(response.model_dump()) + assert session is not None + assert render("claude-sonnet-5", session, tmp_path, use_color=False) == ( + "Routed to: claude-sonnet-5 ยท Savings unavailable" + ) + @pytest.mark.asyncio async def test_another_keys_session_is_a_404_even_for_an_admin(self, monkeypatch: pytest.MonkeyPatch): # The scope is the caller's own key hash, exactly what the spend writer keyed the row under; @@ -1168,14 +1423,13 @@ class TestAutoRouterSession: assert response.router_name == "new-auto" @pytest.mark.asyncio - async def test_a_reconfigured_router_keeps_the_label_the_money_was_priced_against( + async def test_a_reconfigured_router_retains_historical_baseline_labels( self, monkeypatch: pytest.MonkeyPatch - ): - # The proxy's router now prices against a different baseline, but the row's money was priced - # against opus for two of three turns, and the label says so; the full split is on the response. + ) -> None: from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_session - priced = {"anthropic/claude-opus-5": 2, "anthropic/claude-sonnet-5": 1} + priced: Final = {"anthropic/claude-opus-5": 2, "anthropic/claude-sonnet-5": 1} + historical: Final = {"old-baseline": 100, **priced} self._rig( monkeypatch, [ @@ -1183,14 +1437,14 @@ class TestAutoRouterSession: **self.ROW, "api_key": ADMIN.api_key, "session_id": "s", - "baseline_models": {"old-baseline": 100}, + "baseline_models": historical, "savings_estimated_baseline_models": priced, } ], ) - response = await get_auto_router_session(user_api_key_dict=ADMIN, session_id="s") - assert response.baseline_model == "anthropic/claude-opus-5" - assert response.baseline_models == priced + response: Final = await get_auto_router_session(user_api_key_dict=ADMIN, session_id="s") + assert response.baseline_model == "old-baseline" + assert response.baseline_models == historical @pytest.mark.asyncio async def test_an_oversized_client_session_id_is_bounded_like_the_writer_bounded_it( diff --git a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py index baaf3f4ba2f..8af492d9d20 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py +++ b/tests/test_litellm/proxy/management_endpoints/test_common_daily_activity.py @@ -1477,6 +1477,13 @@ _DAILY_USER_SPEND_DDL: Final = """ prompt_caching_savings_spend DOUBLE PRECISION DEFAULT 0, gateway_injected_caching_savings_spend DOUBLE PRECISION DEFAULT 0, autorouter_savings_spend DOUBLE PRECISION DEFAULT 0, + autorouter_accounted_requests BIGINT NOT NULL DEFAULT 0, + autorouter_requests BIGINT NOT NULL DEFAULT 0, + autorouter_llm_spend DOUBLE PRECISION NOT NULL DEFAULT 0, + autorouter_classifier_cost DOUBLE PRECISION NOT NULL DEFAULT 0, + autorouter_classifier_cost_recorded_requests BIGINT NOT NULL DEFAULT 0, + autorouter_estimated_requests BIGINT NOT NULL DEFAULT 0, + autorouter_estimated_actual_spend DOUBLE PRECISION NOT NULL DEFAULT 0, spend DOUBLE PRECISION DEFAULT 0, api_requests BIGINT DEFAULT 0, successful_requests BIGINT DEFAULT 0, diff --git a/tests/test_litellm/proxy/spend_tracking/test_daily_global_spend_rollup.py b/tests/test_litellm/proxy/spend_tracking/test_daily_global_spend_rollup.py index 3da587435ad..6e06a4f2a79 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_daily_global_spend_rollup.py +++ b/tests/test_litellm/proxy/spend_tracking/test_daily_global_spend_rollup.py @@ -385,6 +385,13 @@ _DAILY_USER_SPEND_DDL: Final = """ prompt_caching_savings_spend DOUBLE PRECISION DEFAULT 0, gateway_injected_caching_savings_spend DOUBLE PRECISION DEFAULT 0, autorouter_savings_spend DOUBLE PRECISION DEFAULT 0, + autorouter_accounted_requests BIGINT NOT NULL DEFAULT 0, + autorouter_requests BIGINT NOT NULL DEFAULT 0, + autorouter_llm_spend DOUBLE PRECISION NOT NULL DEFAULT 0, + autorouter_classifier_cost DOUBLE PRECISION NOT NULL DEFAULT 0, + autorouter_classifier_cost_recorded_requests BIGINT NOT NULL DEFAULT 0, + autorouter_estimated_requests BIGINT NOT NULL DEFAULT 0, + autorouter_estimated_actual_spend DOUBLE PRECISION NOT NULL DEFAULT 0, spend DOUBLE PRECISION DEFAULT 0, api_requests BIGINT DEFAULT 0, successful_requests BIGINT DEFAULT 0, diff --git a/tests/test_litellm/proxy/spend_tracking/test_savings.py b/tests/test_litellm/proxy/spend_tracking/test_savings.py index de413a86521..c1251f9a6c5 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_savings.py +++ b/tests/test_litellm/proxy/spend_tracking/test_savings.py @@ -19,6 +19,30 @@ from litellm.types.utils import Usage pytestmark = pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize("estimate, saved, expected", [ + (None, 0.0, 0.0), + (None, -0.5, -0.5), + (None, 0.5, 0.5), + (None, None, None), + (None, True, None), + (None, float("nan"), None), + ({"version": 3, "status": "estimated"}, 0.0, 0.0), + ({"version": 2, "status": "estimated"}, -0.5, -0.5), + ({"version": 1, "status": "unknown"}, 0.5, None), + ({"version": 0, "status": "estimated"}, 0.5, None), +]) +def test_known_autorouter_savings_preserves_zero_and_legacy_provenance( + estimate: dict[str, object] | None, saved: object, expected: float | None, +) -> None: + from litellm.proxy.spend_tracking.savings import known_autorouter_savings + + result: Final = known_autorouter_savings( + model=None, custom_llm_provider=None, routing_decision=None, usage_object=None, + recorded_autorouter_savings=saved, recorded_autorouter_savings_estimate=estimate, + ) + assert result == expected + + @pytest.mark.parametrize("model,usage", [ (None, {"cache_read_input_tokens": 100}), ("claude-sonnet-5", None), diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index b84efda3308..802e05d9d07 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -2266,6 +2266,7 @@ class TestRunServerDbSetup: use_migrate=True, use_v2_resolver=True ) + @pytest.mark.parametrize("coverage_failure", [False, True]) @patch("subprocess.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") @@ -2278,6 +2279,7 @@ class TestRunServerDbSetup: mock_setup_database, mock_atexit_register, mock_subprocess_run, + coverage_failure: bool, ): """Test that proxy exits with code 1 when PrismaManager.setup_database returns False and --enforce_prisma_migration_check is set""" from litellm.proxy.proxy_cli import run_server @@ -2285,6 +2287,9 @@ class TestRunServerDbSetup: mock_subprocess_run.return_value = MagicMock(returncode=0) mock_should_update_schema.return_value = True mock_setup_database.return_value = False + mock_setup_database.side_effect = ( + RuntimeError("daily auto-router coverage installation failed") if coverage_failure else None + ) mock_proxy_module = MagicMock( app=MagicMock(), @@ -2324,13 +2329,13 @@ class TestRunServerDbSetup: [ "--local", "--skip_server_startup", - "--enforce_prisma_migration_check", + "--use_prisma_db_push" if coverage_failure else "--enforce_prisma_migration_check", ], standalone_mode=False, ) - assert exc_info.value.code == 1 + assert exc_info.value.code == (2 if coverage_failure else 1) mock_setup_database.assert_called_once_with( - use_migrate=True, use_v2_resolver=True + use_migrate=not coverage_failure, use_v2_resolver=True ) @patch("subprocess.run") diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx index a144630cdd0..918f4587ab6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx @@ -1,5 +1,6 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { fireEvent, render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import React from "react"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -68,6 +69,9 @@ const totals = (overrides: Partial = {}): Totals => ({ avg_session_seconds: 7560, avg_tokens_per_session: 5_300_000, spend: 359.86, + llm_spend: (overrides.spend ?? 359.86) - (overrides.classifier_cost ?? 6.146), + cost_coverage: "complete", + cost_requests: overrides.turns ?? 3073, savings_estimated_turns: overrides.turns ?? 3073, savings_estimated_actual_spend: overrides.spend ?? 359.86, classifier_cost: 6.146, @@ -102,6 +106,9 @@ const zeroTotals: Totals = { avg_session_seconds: 0, avg_tokens_per_session: 0, spend: 0, + llm_spend: 0, + cost_coverage: "complete", + cost_requests: 0, savings_estimated_turns: 0, savings_estimated_actual_spend: 0, classifier_cost: 0, @@ -116,6 +123,7 @@ const group = (overrides: Partial = {}): AutoRouterBen router_name: "claude-auto", router_type: "complexity", ...totals(), + cost_requests: null, ...overrides, }); @@ -157,37 +165,28 @@ describe("AutoRouterBenchmarksTab", () => { mockAutoRouters(); }); - it.each([ - { estimatedTurns: 0, saved: null, pct: null }, - { estimatedTurns: 10, saved: -0.5, pct: -33.3 }, - { estimatedTurns: 10, saved: 0, pct: 0 }, - ])("preserves costs for $estimatedTurns estimated turns with savings $saved", ({ estimatedTurns, saved, pct }) => { - const cohort = { + it.each([0, 2])("retains historical savings when only %i turns have current estimates", (estimatedTurns) => { + const history: Partial = { savings_estimated_turns: estimatedTurns, - savings_estimated_actual_spend: estimatedTurns ? 2 : 0, - saved_spend: saved, - baseline_spend: estimatedTurns ? 2 + (saved ?? 0) : null, - saved_pct: pct, - saved_per_session: null, + savings_estimated_actual_spend: estimatedTurns ? 0.0004 : 0, + cost_coverage: "partial", + cost_requests: null, + baseline_spend: null, + saved_pct: null, }; - const partial = totals(cohort); + const partial = totals(history); mockHook({ data: response([], partial) }); renderTab(); - expect(screen.getByText("Estimated savings on covered turns")).toBeInTheDocument(); - expect(screen.getByText(`${estimatedTurns} of 3,073 turns estimated`)).toBeInTheDocument(); + expect(screen.getByText("Total estimated savings")).toBeInTheDocument(); + expect(screen.getByText("$2,174.59")).toBeInTheDocument(); + expect(screen.queryByText("-86%")).not.toBeInTheDocument(); expect(screen.getByText("$359.86")).toBeInTheDocument(); - expect(screen.getByText("Actual spend on covered turns")).toBeInTheDocument(); - expect(screen.getByText("Estimated baseline spend on covered turns")).toBeInTheDocument(); - expect(screen.getAllByText("Unavailable")).toHaveLength(estimatedTurns ? 1 : 3); - if (saved === 0) { - expect(screen.getByText("0%")).toBeInTheDocument(); - expect(screen.getAllByText("$2.00")).toHaveLength(2); - } else if (estimatedTurns) { - expect(screen.getByText("-$0.5000")).toBeInTheDocument(); - expect(screen.getByText("+33%")).toBeInTheDocument(); - } else { - expect(screen.queryByText("+0%")).not.toBeInTheDocument(); - } + expect(screen.getByText("Unavailable")).toBeInTheDocument(); + expect(screen.getByText("$23.13")).toBeInTheDocument(); + expect(screen.getByText("Estimated baseline spend")).toBeInTheDocument(); + expect(screen.queryByText(/turns estimated/)).not.toBeInTheDocument(); + expect(screen.queryByText(/older estimates.*excluded|excluded.*older estimates/)).not.toBeInTheDocument(); + expect(screen.queryByText("Actual spend on covered turns")).not.toBeInTheDocument(); }); it("leads with total estimated savings, before the four session-shape metrics", () => { @@ -216,13 +215,155 @@ describe("AutoRouterBenchmarksTab", () => { expect(screen.getByText("-86%")).toBeInTheDocument(); expect(screen.getByText("Actual auto-router spend")).toBeInTheDocument(); expect(screen.getByText("$359.86")).toBeInTheDocument(); - expect(screen.getByText("Estimated spend at highest-tier model")).toBeInTheDocument(); + expect(screen.getByText("Estimated baseline spend")).toBeInTheDocument(); expect(screen.getByText("$2,534.45")).toBeInTheDocument(); expect(screen.getByText("32.7")).toBeInTheDocument(); expect(screen.getByText("2.1h")).toBeInTheDocument(); expect(screen.getByText("5.3M")).toBeInTheDocument(); }); + it("keeps daily savings visible when matching historical costs are unavailable", () => { + const dailyValues: Partial = { + spend: null, + llm_spend: null, + cost_coverage: "unavailable", + cost_requests: null, + classifier_cost: null, + baseline_spend: null, + saved_pct: null, + }; + const daily = totals(dailyValues); + mockHook({ data: response([group()], daily) }); + renderTab(); + + expect(screen.getByText("$2,174.59")).toBeInTheDocument(); + expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([ + "Unavailable", + "Unavailable", + "Unavailable", + "Unavailable", + ]); + expect(screen.getAllByText("Unavailable")).toHaveLength(4); + expect(screen.getByText("$23.13")).toBeInTheDocument(); + expect(screen.queryByText("-86%")).not.toBeInTheDocument(); + expect(screen.queryByText(/\/ 1K turns/)).not.toBeInTheDocument(); + expect(screen.getByText("Whole sessions overlapping the selected dates")).toBeInTheDocument(); + }); + + it("shows recorded historical costs without presenting partial costs as the full savings comparison", () => { + const recordedCosts: Partial = { + spend: 12, + llm_spend: 10, + classifier_cost: null, + cost_coverage: "partial", + cost_requests: null, + baseline_spend: null, + saved_pct: null, + }; + mockHook({ data: response([], totals(recordedCosts)) }); + renderTab(); + + expect(screen.getByText("$2,174.59")).toBeInTheDocument(); + expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([ + "$12.00", + "$10.00", + "Unavailable", + "Unavailable", + ]); + expect(screen.getByText("Partial")).toBeInTheDocument(); + expect(screen.getByText("Some request costs are unavailable")).toBeInTheDocument(); + expect(screen.queryByText("-86%")).not.toBeInTheDocument(); + expect(screen.queryByText(/\/ 1K turns/)).not.toBeInTheDocument(); + }); + + it("separates actual spend from the estimated-request comparison and uses request-date classification counts", () => { + const requestCosts: Partial = { + spend: 12, + llm_spend: 10, + classifier_cost: 2, + cost_requests: 4, + savings_estimated_turns: 2, + savings_estimated_actual_spend: 4, + saved_spend: 6, + baseline_spend: 10, + saved_pct: 60, + }; + mockHook({ data: response([], totals(requestCosts)) }); + renderTab(); + + expect(screen.getByText("$6.00")).toBeInTheDocument(); + expect(screen.getByText("-60%")).toBeInTheDocument(); + expect(screen.getByText("2 of 4 requests have savings estimates")).toBeInTheDocument(); + expect(screen.getByText("Actual spend on estimated requests")).toBeInTheDocument(); + expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([ + "$12.00", + "$10.00", + "$2.00", + "$4.00", + "$10.00", + ]); + expect(screen.getByText("($500.00 / 1K turns)")).toBeInTheDocument(); + }); + + it("labels a selected router as session usage and keeps its recorded cost breakdown", async () => { + const user = userEvent.setup(); + const dailyValues: Partial = { + saved_spend: 3_000, + spend: null, + llm_spend: null, + cost_coverage: "unavailable", + cost_requests: null, + classifier_cost: null, + baseline_spend: null, + saved_pct: null, + }; + const daily = totals(dailyValues); + mockHook({ data: response([group()], daily) }); + renderTab(); + + expect(screen.getByText("$3,000.00")).toBeInTheDocument(); + await user.click(screen.getByRole("combobox")); + await user.click(await screen.findByRole("option", { name: "claude-auto" })); + + expect(screen.getByRole("heading", { name: "Auto-router session usage" })).toBeInTheDocument(); + expect(screen.getByText("Whole sessions overlapping the selected dates")).toBeInTheDocument(); + expect(screen.getByText("$2,174.59")).toBeInTheDocument(); + expect(screen.queryByText("$3,000.00")).not.toBeInTheDocument(); + expect(screen.getByText("-86%")).toBeInTheDocument(); + expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([ + "$359.86", + "$353.71", + "$6.15", + "$2,534.45", + ]); + }); + + it("keeps selected-router spend visible without a baseline or percentage for unestimated turns", async () => { + const user = userEvent.setup(); + const unknownSavings: Partial = { + savings_estimated_turns: 0, + savings_estimated_actual_spend: 0, + saved_spend: 0, + baseline_spend: null, + saved_pct: null, + saved_per_session: 0, + }; + mockHook({ data: response([group(unknownSavings)]) }); + renderTab(); + + await user.click(screen.getByRole("combobox")); + await user.click(await screen.findByRole("option", { name: "claude-auto" })); + + expect(screen.getByRole("heading", { name: "Auto-router session usage" })).toBeInTheDocument(); + expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([ + "$359.86", + "$353.71", + "$6.15", + "Unavailable", + ]); + expect(screen.queryByText("0%")).not.toBeInTheDocument(); + }); + it.each([ { spend: 20665.28, classifier_cost: 342.18, turns: 140815, llm: "$20,323.10", cost: "$342.18", rate: "$2.43" }, { spend: 0, classifier_cost: 0, turns: 0, llm: "$0.00", cost: "$0.00", rate: "$0.00" }, @@ -243,15 +384,31 @@ describe("AutoRouterBenchmarksTab", () => { }); it.each([null, undefined])("keeps totals when the classification breakdown is %s", (classifier_cost) => { - const stats = totals({ classifier_cost }); + const partialCosts: Partial = { + classifier_cost, + cost_coverage: "partial", + baseline_spend: null, + saved_pct: null, + }; + const stats = totals(partialCosts); mockHook({ data: response([group(stats)], stats) }); renderTab(); expect(screen.getAllByText("Unavailable")).toHaveLength(2); expect(screen.queryByText(/\/ 1K turns/)).not.toBeInTheDocument(); expect(screen.getByText("$359.86")).toBeInTheDocument(); + expect(screen.getByText("$353.71")).toBeInTheDocument(); expect(screen.getByText("$2,174.59")).toBeInTheDocument(); - expect(screen.getByText(/some usage predates classification-cost tracking/)).toBeInTheDocument(); + }); + + it.each([ + { llm_spend: undefined, displayed: "$353.71" }, + { llm_spend: null, displayed: "Unavailable" }, + ])("only derives LLM spend when an older proxy omits its subtotal: %j", ({ llm_spend, displayed }) => { + mockHook({ data: response([], totals({ llm_spend })) }); + renderTab(); + + expect(screen.getAllByRole("definition")[1]).toHaveTextContent(displayed); }); it("pairs the savings with the session count it was earned over, in its own tile", () => { @@ -275,7 +432,7 @@ describe("AutoRouterBenchmarksTab", () => { "Actual auto-router spend", "LLM spend", "Classification cost($2.00 / 1K turns)", - "Estimated spend at highest-tier model", + "Estimated baseline spend", ]); expect(values).toEqual(["$359.86", "$353.71", "$6.15", "$2,534.45"]); }); @@ -441,7 +598,11 @@ describe("AutoRouterBenchmarksTab", () => { expect(onDateChange).toHaveBeenCalledWith({ from: new Date(2026, 7, 1), to: new Date(2026, 7, 5) }); }); - it("scopes the query to one key when the usage view is mounted for a key", () => { + it.each([ + { apiKey: "key-hash-1", userId: undefined }, + { apiKey: undefined, userId: "user-1" }, + { apiKey: "key-hash-1", userId: "user-1" }, + ])("preserves the key/user filters for the usage view: %j", ({ apiKey, userId }) => { mockHook({ data: response([group()]) }); const dateValue = { from: new Date(2026, 6, 6), to: new Date(2026, 7, 5) }; const activity = { @@ -456,11 +617,11 @@ describe("AutoRouterBenchmarksTab", () => { }; render( - + , ); - expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, "key-hash-1", undefined); + expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, apiKey, userId); expect(screen.getByText("Total estimated savings")).toBeInTheDocument(); expect(screen.queryByRole("tab", { name: "Shadow Evals" })).not.toBeInTheDocument(); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx index 063598bd46e..ced592c073e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx @@ -25,6 +25,7 @@ import { groupLabel, pctLabel, viewFor, + viewGroup, type AutoRouterBenchmarksResponse, type AutoRouterCacheStats, type BenchmarkView, @@ -71,16 +72,60 @@ const SpendRow: React.FC<{ label: string; value: string; hint?: string; subdued? ); +const CostBreakdown: React.FC<{ view: BenchmarkView; partialComparison: boolean }> = ({ view, partialComparison }) => { + const stats = view.stats; + const costTurns = viewGroup(view) ? stats.turns : stats.cost_requests; + const llmSpend = + stats.llm_spend === undefined && stats.spend != null && stats.classifier_cost != null + ? stats.spend - stats.classifier_cost + : stats.llm_spend; + return ( +
+ +
+ + +
+ {stats.cost_coverage === "partial" && ( +

Some request costs are unavailable

+ )} + + {partialComparison && stats.baseline_spend != null && ( + + )} + +
+ ); +}; + const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => { const stats = view.stats; const cheaper = stats.saved_spend != null && stats.saved_spend >= 0; - const completeCoverage = stats.savings_estimated_turns === stats.turns; + const dailyCosts = viewGroup(view) === null; + const partialComparison = + dailyCosts && stats.cost_requests != null && stats.savings_estimated_turns < stats.cost_requests; return (

- {completeCoverage ? "Total estimated savings" : "Estimated savings on covered turns"} + Total estimated savings

@@ -96,51 +141,15 @@ const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => { )}

-

- {stats.savings_estimated_turns.toLocaleString()} of {stats.turns.toLocaleString()} turns estimated -

- {!completeCoverage && ( + {partialComparison && (

- Turns without a current estimate are excluded, including older estimates. + {stats.savings_estimated_turns.toLocaleString()} of {stats.cost_requests?.toLocaleString()} requests have + savings estimates

)}
-
- -
- - -
- {stats.classifier_cost == null && ( -

- Breakdown unavailable because some usage predates classification-cost tracking. -

- )} - - {!completeCoverage && ( - - )} - -
+
); @@ -293,6 +302,9 @@ const BenchmarksBody: React.FC = ({ isPending, error, data, <> + {!viewGroup(view) && ( +

Whole sessions overlapping the selected dates

+ )}
@@ -306,15 +318,6 @@ const BenchmarksBody: React.FC = ({ isPending, error, data,
-

- Compares covered turns with the estimated cost of using the router's highest-tier baseline model. Estimates - use registered requests since tracking began, matching cache prefixes and expiry, and the actual response - length. Total actual spend includes every turn; savings and baseline spend include only turns with a current - estimate, including turns with zero savings. Savings are net of recorded LLM classification cost. Classification - cost per 1K turns is averaged over all auto-router turns, including those that skip classification. The range - counts whole sessions that overlap it, so totals can differ from savings views that group usage by UTC day. -

-

Auto-router prompt caching

@@ -347,15 +350,22 @@ export const AutoRouterUsageView: React.FC = ({ const { data: autoRouters } = useAutoRouters(); const groups = data?.groups ?? []; - const selectedLabel = data ? viewFor(data, selectedKey).label : "All auto-routers"; + const selectedView = data ? viewFor(data, selectedKey) : null; + const selectedLabel = selectedView?.label ?? "All auto-routers"; + const selectedGroup = selectedView ? viewGroup(selectedView) : null; const rangeLabel = formatRangeLabel(dateValue.from, dateValue.to); return (
-

Auto-router usage

+

+ {selectedGroup ? "Auto-router session usage" : "Auto-router usage"} +

{rangeLabel &&

{rangeLabel} (UTC)

} + {selectedGroup && ( +

Whole sessions overlapping the selected dates

+ )}
@@ -379,8 +389,7 @@ export const AutoRouterUsageView: React.FC = ({ {userId && (

- Usage for this user across API keys and JWT-authenticated requests. Older sessions recorded without a user ID - are not included. + Usage for this user across API keys and JWT-authenticated requests.

)} ({ })); import TierTurnsChart, { tierDisplayLabel } from "./TierTurnsChart"; -import type { AutoRouterBenchmarkGroup, BenchmarkView } from "./autoRouterBenchmarks"; +import type { AutoRouterBenchmarkGroup, AutoRouterBenchmarkTotals, BenchmarkView } from "./autoRouterBenchmarks"; -const totalsOnly = { +const totalsOnly: AutoRouterBenchmarkTotals = { sessions: 3, turns: 9, avg_turns_per_session: 3, avg_session_seconds: 60, avg_tokens_per_session: 100, spend: 1, + classifier_cost: 0, + llm_spend: 1, + cost_coverage: "complete", + cost_requests: null, savings_estimated_turns: 9, savings_estimated_actual_spend: 1, saved_spend: 1, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.test.ts index 0586163e77e..b4325254b07 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks.test.ts @@ -37,6 +37,9 @@ const totals = (overrides: Partial = {}) => ({ avg_session_seconds: 7560, avg_tokens_per_session: 5_300_000, spend: 359.86, + llm_spend: 353.714, + cost_coverage: "complete" as const, + cost_requests: null, savings_estimated_turns: overrides.turns ?? 3073, savings_estimated_actual_spend: overrides.spend ?? 359.86, classifier_cost: 6.146, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx index 6a0e55a6dda..f734de07e4c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/users/_components/view_users/user_info_view.integration.test.tsx @@ -342,14 +342,17 @@ const routerUsageResponse = (saved: number): AutoRouterBenchmarksResponse => ({ avg_turns_per_session: 1, avg_session_seconds: 0, avg_tokens_per_session: 100, - spend: 10, + spend: null, + llm_spend: null, + cost_coverage: "unavailable", + cost_requests: null, savings_estimated_turns: 2, savings_estimated_actual_spend: 10, - classifier_cost: 0, + classifier_cost: null, saved_spend: saved, - baseline_spend: 10 + saved, - saved_pct: (100 * saved) / (10 + saved), - saved_per_session: saved / 2, + baseline_spend: null, + saved_pct: null, + saved_per_session: 3, cache: { coverage_pct: 100, hit_rate_pct: 0, @@ -405,7 +408,17 @@ describe("UserInfoView auto-router usage", () => { const params = new URL(request.url).searchParams; expect(params.get("user_id")).toBe("user-123"); expect(params.has("api_key")).toBe(false); - expect(screen.getByText(/Older sessions recorded without a user ID are not included/)).toBeInTheDocument(); + expect( + screen.getByText("Usage for this user across API keys and JWT-authenticated requests."), + ).toBeInTheDocument(); + expect(screen.getByText("$3.00")).toBeInTheDocument(); + expect(screen.getByText("Estimated baseline spend")).toBeInTheDocument(); + expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([ + "Unavailable", + "Unavailable", + "Unavailable", + "Unavailable", + ]); }, ); diff --git a/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx b/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx index 5429c688e17..f10b27e4b7d 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyAutoRouterUsageTab.integration.test.tsx @@ -1,6 +1,8 @@ import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { AutoRouterBenchmarksResponse } from "@/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks"; import { renderWithProviders, testQueryClient } from "../../../tests/test-utils"; import KeyAutoRouterUsageTab from "./KeyAutoRouterUsageTab"; @@ -32,6 +34,9 @@ const stats = { avg_session_seconds: 30, avg_tokens_per_session: 100, spend: 1.25, + llm_spend: 1, + cost_coverage: "complete" as const, + cost_requests: null, savings_estimated_turns: 4, savings_estimated_actual_spend: 1.25, classifier_cost: 0.25, @@ -42,11 +47,20 @@ const stats = { cache, }; -const benchmarks = { +const benchmarks: AutoRouterBenchmarksResponse = { start_date: "2025-01-01", end_date: "2025-01-31", routers_in_scope: 2, - totals: stats, + totals: { + ...stats, + saved_spend: 12.75, + spend: null, + llm_spend: null, + cost_coverage: "unavailable", + classifier_cost: null, + baseline_spend: null, + saved_pct: null, + }, groups: [ { router_name: "router-one", router_type: "complexity", tier_turns: { SIMPLE: 4 }, ...stats }, { @@ -55,6 +69,7 @@ const benchmarks = { tier_turns: { SIMPLE: 1 }, ...stats, spend: 0.25, + llm_spend: 0, saved_spend: 0.75, baseline_spend: 1, }, @@ -77,14 +92,38 @@ describe("KeyAutoRouterUsageTab", () => { vi.stubGlobal("fetch", fetchMock); }); - it("renders this key's spend, baseline, savings and per-router filter", async () => { + it("renders this key's daily savings and selected-router session costs", async () => { + const user = userEvent.setup(); const activity = { dateValue: { from: new Date(2025, 0, 1), to: new Date(2025, 0, 31) }, onDateChange: vi.fn(), }; renderWithProviders(); - expect(await screen.findByText("$8.75")).toBeInTheDocument(); + expect(await screen.findByText("$12.75")).toBeInTheDocument(); + expect(screen.getAllByRole("definition").map((node) => node.textContent)).toEqual([ + "Unavailable", + "Unavailable", + "Unavailable", + "Unavailable", + ]); + expect(screen.getByText("$4.38")).toBeInTheDocument(); + expect(screen.queryByText("-88%")).not.toBeInTheDocument(); + expect(screen.getByText("All auto-routers")).toBeInTheDocument(); + + const benchmarkUrl = new URL(requestedUrls().find((url) => url.includes("/auto_router/benchmarks")) ?? ""); + expect(benchmarkUrl.searchParams.get("api_key")).toBe("key-hash-1"); + expect(benchmarkUrl.searchParams.get("start_date")).toBe("2025-01-01"); + expect(benchmarkUrl.searchParams.get("end_date")).toBe("2025-01-31"); + + await user.click(screen.getByRole("combobox")); + await user.click(await screen.findByRole("option", { name: "router-one" })); + + expect(screen.getByRole("heading", { name: "Auto-router session usage" })).toBeInTheDocument(); + expect(screen.getByText("Whole sessions overlapping the selected dates")).toBeInTheDocument(); + expect(screen.getByText("$8.75")).toBeInTheDocument(); + expect(screen.queryByText("$12.75")).not.toBeInTheDocument(); + expect(screen.getByText("-88%")).toBeInTheDocument(); expect(screen.getByText("Actual auto-router spend")).toBeInTheDocument(); expect(screen.getByText("$1.25")).toBeInTheDocument(); expect(screen.getByText("LLM spend")).toBeInTheDocument(); @@ -92,15 +131,9 @@ describe("KeyAutoRouterUsageTab", () => { expect(screen.getByText("Classification cost")).toBeInTheDocument(); expect(screen.getByText("$0.2500")).toBeInTheDocument(); expect(screen.getByText("($62.50 / 1K turns)")).toBeInTheDocument(); - expect(screen.getByText("Estimated spend at highest-tier model")).toBeInTheDocument(); + expect(screen.getByText("Estimated baseline spend")).toBeInTheDocument(); expect(screen.getByText("$10.00")).toBeInTheDocument(); expect(screen.getByText("Auto-router prompt caching")).toBeInTheDocument(); expect(screen.getAllByText("50.0%").length).toBeGreaterThan(0); - expect(screen.getByText("All auto-routers")).toBeInTheDocument(); - - const benchmarkUrl = new URL(requestedUrls().find((url) => url.includes("/auto_router/benchmarks")) ?? ""); - expect(benchmarkUrl.searchParams.get("api_key")).toBe("key-hash-1"); - expect(benchmarkUrl.searchParams.get("start_date")).toBe("2025-01-01"); - expect(benchmarkUrl.searchParams.get("end_date")).toBe("2025-01-31"); }); }); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ebd87b30901..17f0cdba8a9 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -1263,12 +1263,12 @@ export interface paths { * @description Benchmarks for the auto-router dashboard: session shape, savings against the configured * baseline, and prompt-caching behaviour bucketed by what the router did. * - * Reads session rollups folded once per request at spend-write time, so this endpoint - * never scans LiteLLM_SpendLogs. A user filter selects only turns attributed to that - * internal user when written; older key-only history remains outside user views. A session - * is in the window when it overlaps it: its last turn is on or after start_date and its first turn is on or before - * end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is - * over that bucket's turns. + * Total savings use the same request-date daily aggregation as Overall cost optimization, + * including recorded history and requests without session IDs. Costs use matching request + * dates with explicit coverage for history predating daily router cost tracking. Session + * statistics and per-router groups cover whole sessions that overlap the window. + * A user filter uses the user recorded when each row was written. + * Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is over its turns. * * The rollup supplies the measures, never the list. Which routers appear comes from the * model registry, so one shows up as soon as it is configured and reads zero until it @@ -24180,7 +24180,7 @@ export interface components { }; /** * AutoRouterBenchmarkGroup - * @description One auto-router's slice of the benchmarks. + * @description One auto-router's whole overlapping sessions, separate from request-date daily totals. */ AutoRouterBenchmarkGroup: { /** Avg Session Seconds */ @@ -24191,15 +24191,32 @@ export interface components { avg_turns_per_session: number; /** * Baseline Spend - * @description Estimated single-model cost for covered turns only + * @description Matching actual spend plus recorded savings. Daily totals compare estimated requests only; null when historical cost coverage or the savings estimate is unavailable */ baseline_spend: number | null; cache: components["schemas"]["AutoRouterCacheStats"]; /** * Classifier Cost - * @description Recorded LLM classifier cost already included in spend; null when any session turns predate subtotal recording, and zero for an empty window + * @description Classifier charges included in actual cost; null when any contributing request lacks the subtotal */ classifier_cost: number | null; + /** + * Cost Coverage + * @description Whether recorded costs cover the entire selected population + * @default unavailable + * @enum {string} + */ + cost_coverage: "complete" | "partial" | "unavailable"; + /** + * Cost Requests + * @description Number of requests in the cost population; null when historical coverage is incomplete + */ + cost_requests?: number | null; + /** + * Llm Spend + * @description Routed LLM cost excluding classifier charges + */ + llm_spend?: number | null; /** * Router Name * @description The auto-router alias requests were sent to @@ -24212,36 +24229,36 @@ export interface components { router_type: string; /** * Saved Pct - * @description Covered savings over covered baseline spend, as a percentage + * @description Recorded savings divided by the matching estimated baseline; null when that baseline is unavailable */ saved_pct: number | null; /** * Saved Per Session - * @description Average session savings; unavailable unless every turn is covered + * @description Recorded savings in whole overlapping sessions divided by their session count, independent of request-date daily savings totals */ saved_per_session: number | null; /** * Saved Spend - * @description Signed savings for covered turns only; null when traffic has no current estimates + * @description Signed recorded savings, including historical and current estimates once. Totals use the same request-date daily source as Overall cost optimization, including requests without sessions. Per-router groups cover whole overlapping sessions. Turns without an estimate add no savings */ saved_spend: number | null; /** * Savings Estimated Actual Spend - * @description Actual spend, including classifier cost, for covered turns only + * @description Actual spend including classification for requests covered by the savings comparison */ savings_estimated_actual_spend: number; /** * Savings Estimated Turns - * @description Turns covered by the current savings estimator; legacy estimates are excluded + * @description Requests with an available savings comparison. Totals use request dates and groups use session coverage. Historical savings remain included even when their coverage was not tracked */ savings_estimated_turns: number; /** Sessions */ sessions: number; /** * Spend - * @description What the routed traffic actually cost + * @description Actual auto-router cost including classification. Totals use request dates; groups cover whole overlapping sessions. Partial recorded history is identified by cost_coverage */ - spend: number; + spend: number | null; /** * Tier Turns * @description Turns per tier, keyed by the tier name the routing decision recorded at request time (never re-derived at read time, since the tier-to-model mapping is mutable config). Tier names are scoped to this group's router_type and are not comparable across types: a complexity router reports 'SIMPLE'/'MEDIUM'/'COMPLEX'/'REASONING', a quality router reports its numeric quality tier, and an adaptive router records no tier at all. Turns no tier served (the classifier fell back to default_model) are absent rather than pooled under a sentinel key, so the values may sum to less than turns @@ -24254,7 +24271,7 @@ export interface components { }; /** * AutoRouterBenchmarkTotals - * @description Session-shape and savings aggregates over auto-routed traffic in the window. + * @description Request-date savings totals with statistics for whole sessions overlapping the window. */ AutoRouterBenchmarkTotals: { /** Avg Session Seconds */ @@ -24265,53 +24282,70 @@ export interface components { avg_turns_per_session: number; /** * Baseline Spend - * @description Estimated single-model cost for covered turns only + * @description Matching actual spend plus recorded savings. Daily totals compare estimated requests only; null when historical cost coverage or the savings estimate is unavailable */ baseline_spend: number | null; cache: components["schemas"]["AutoRouterCacheStats"]; /** * Classifier Cost - * @description Recorded LLM classifier cost already included in spend; null when any session turns predate subtotal recording, and zero for an empty window + * @description Classifier charges included in actual cost; null when any contributing request lacks the subtotal */ classifier_cost: number | null; + /** + * Cost Coverage + * @description Whether recorded costs cover the entire selected population + * @default unavailable + * @enum {string} + */ + cost_coverage: "complete" | "partial" | "unavailable"; + /** + * Cost Requests + * @description Number of requests in the cost population; null when historical coverage is incomplete + */ + cost_requests?: number | null; + /** + * Llm Spend + * @description Routed LLM cost excluding classifier charges + */ + llm_spend?: number | null; /** * Saved Pct - * @description Covered savings over covered baseline spend, as a percentage + * @description Recorded savings divided by the matching estimated baseline; null when that baseline is unavailable */ saved_pct: number | null; /** * Saved Per Session - * @description Average session savings; unavailable unless every turn is covered + * @description Recorded savings in whole overlapping sessions divided by their session count, independent of request-date daily savings totals */ saved_per_session: number | null; /** * Saved Spend - * @description Signed savings for covered turns only; null when traffic has no current estimates + * @description Signed recorded savings, including historical and current estimates once. Totals use the same request-date daily source as Overall cost optimization, including requests without sessions. Per-router groups cover whole overlapping sessions. Turns without an estimate add no savings */ saved_spend: number | null; /** * Savings Estimated Actual Spend - * @description Actual spend, including classifier cost, for covered turns only + * @description Actual spend including classification for requests covered by the savings comparison */ savings_estimated_actual_spend: number; /** * Savings Estimated Turns - * @description Turns covered by the current savings estimator; legacy estimates are excluded + * @description Requests with an available savings comparison. Totals use request dates and groups use session coverage. Historical savings remain included even when their coverage was not tracked */ savings_estimated_turns: number; /** Sessions */ sessions: number; /** * Spend - * @description What the routed traffic actually cost + * @description Actual auto-router cost including classification. Totals use request dates; groups cover whole overlapping sessions. Partial recorded history is identified by cost_coverage */ - spend: number; + spend: number | null; /** Turns */ turns: number; }; /** * AutoRouterBenchmarksResponse - * @description Benchmarks for the auto-router dashboard, aggregated from the per-session rollup. + * @description Daily recorded savings with tracked-session breakdowns for the auto-router dashboard. */ AutoRouterBenchmarksResponse: { /** @@ -24573,19 +24607,19 @@ export interface components { AutoRouterSessionResponse: { /** * Baseline Model - * @description The savings baseline most covered turns were priced against, recorded turn by turn, so it still names the counterfactual after the router is reconfigured or removed. None when no turn recorded one: rows from before the baseline was recorded, and adaptive and quality routers, which derive no baseline and so report no savings + * @description The savings baseline recorded by most session turns, including historical turns, recorded turn by turn, so it still names the counterfactual after the router is reconfigured or removed. None when no turn recorded one: rows from before the baseline was recorded, and adaptive and quality routers, which derive no baseline and so report no savings */ baseline_model: string | null; /** * Baseline Models - * @description Covered turns priced against each baseline model; more than one entry means the router's baseline changed mid-session and baseline_spend mixes both + * @description Session turns recording each baseline model, including historical turns; more than one entry means the router's baseline changed mid-session. These counts do not imply savings-estimate coverage */ baseline_models: { [key: string]: number; }; /** * Baseline Spend - * @description Estimated single-model cost; unavailable unless every turn is covered + * @description Actual session spend plus recorded savings; turns without an estimate add no savings */ baseline_spend: number | null; /** @@ -24605,7 +24639,7 @@ export interface components { router_type: string; /** * Saved Spend - * @description Estimated savings for covered turns only, net of classifier cost + * @description Recorded signed savings, retaining historical estimates and adding current estimates once */ saved_spend: number | null; /**