mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
feat(spend): add a per-session auto-router benchmarks rollup
Fold each auto-routed turn into a per-(api_key, session, auto-router) row when it happens, and serve the benchmarks dashboard by summing those rows. Nothing in the feature reads LiteLLM_SpendLogs. The row carries what each tier the session used left in the prompt cache, so a turn's bucket is a question about one model's own record and the upsert answers it against the row it is already writing. Absent from that record means a first visit; present means warm or expired, on one comparison of the idle gap against the TTL the entry was written with. Nothing is read before the write, so there is no state to load or validate, and the statement is atomic.
This commit is contained in:
Tin Chi Lo
parent
5045a576ad
commit
2e2b2624ed
17 changed files with 1602 additions and 7 deletions
|
|
@ -44,6 +44,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
|
|||
"/router/",
|
||||
"/router_settings",
|
||||
"/adaptive_router/",
|
||||
"/auto_router/",
|
||||
"/fallback",
|
||||
"/fallbacks",
|
||||
"/cache_settings",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterSession" (
|
||||
api_key TEXT NOT NULL,
|
||||
session_id TEXT NOT NULL,
|
||||
model_group TEXT NOT NULL,
|
||||
router_kind TEXT NOT NULL,
|
||||
baseline_model TEXT,
|
||||
turns INTEGER NOT NULL DEFAULT 0,
|
||||
turns_with_usage INTEGER NOT NULL DEFAULT 0,
|
||||
total_tokens BIGINT NOT NULL DEFAULT 0,
|
||||
spend DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
baseline_spend DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
first_visit_turns INTEGER NOT NULL DEFAULT 0,
|
||||
first_visit_hits INTEGER NOT NULL DEFAULT 0,
|
||||
warm_turns INTEGER NOT NULL DEFAULT 0,
|
||||
warm_hits INTEGER NOT NULL DEFAULT 0,
|
||||
expired_turns INTEGER NOT NULL DEFAULT 0,
|
||||
expired_hits INTEGER NOT NULL DEFAULT 0,
|
||||
unordered_turns INTEGER NOT NULL DEFAULT 0,
|
||||
unordered_hits INTEGER NOT NULL DEFAULT 0,
|
||||
ephemeral_1h_turns INTEGER NOT NULL DEFAULT 0,
|
||||
tiers JSONB NOT NULL DEFAULT '{}',
|
||||
first_turn_at TIMESTAMP(3) NOT NULL,
|
||||
last_turn_at TIMESTAMP(3) NOT NULL,
|
||||
updated_at TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
CONSTRAINT "LiteLLM_AutoRouterSession_pkey" PRIMARY KEY (api_key, session_id, model_group)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "idx_auto_router_session_started"
|
||||
ON "LiteLLM_AutoRouterSession" (first_turn_at);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "idx_auto_router_session_activity"
|
||||
ON "LiteLLM_AutoRouterSession" (last_turn_at);
|
||||
|
|
@ -1391,6 +1391,41 @@ model LiteLLM_AdaptiveRouterSession {
|
|||
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterSession {
|
||||
api_key String
|
||||
session_id String
|
||||
model_group String
|
||||
router_kind String
|
||||
baseline_model String?
|
||||
|
||||
turns Int @default(0)
|
||||
turns_with_usage Int @default(0)
|
||||
total_tokens BigInt @default(0)
|
||||
spend Float @default(0)
|
||||
baseline_spend Float @default(0)
|
||||
|
||||
first_visit_turns Int @default(0)
|
||||
first_visit_hits Int @default(0)
|
||||
warm_turns Int @default(0)
|
||||
warm_hits Int @default(0)
|
||||
expired_turns Int @default(0)
|
||||
expired_hits Int @default(0)
|
||||
unordered_turns Int @default(0)
|
||||
unordered_hits Int @default(0)
|
||||
|
||||
ephemeral_1h_turns Int @default(0)
|
||||
|
||||
tiers Json @default("{}")
|
||||
|
||||
first_turn_at DateTime
|
||||
last_turn_at DateTime
|
||||
updated_at DateTime @default(now()) @updatedAt
|
||||
|
||||
@@id([api_key, session_id, model_group])
|
||||
@@index([first_turn_at], map: "idx_auto_router_session_started")
|
||||
@@index([last_turn_at], map: "idx_auto_router_session_activity")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workflow Run Tracking
|
||||
//
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import (
|
|||
ToolDiscoveryQueue,
|
||||
)
|
||||
from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
|
||||
from litellm.proxy.spend_tracking.auto_router_sessions import AutoRouterSessionQueue, build_turn_facts
|
||||
from litellm.proxy.spend_tracking.compression_savings import (
|
||||
extract_compression_saved_tokens,
|
||||
)
|
||||
|
|
@ -132,6 +133,46 @@ class DBSpendUpdateWriter:
|
|||
self.daily_agent_spend_update_queue = DailySpendUpdateQueue()
|
||||
self.daily_org_spend_update_queue = DailySpendUpdateQueue()
|
||||
self.daily_tag_spend_update_queue = DailySpendUpdateQueue()
|
||||
self.auto_router_session_queue = AutoRouterSessionQueue()
|
||||
|
||||
def _enqueue_auto_router_turn(self, payload: SpendLogsPayload) -> None:
|
||||
"""Stage one auto-routed turn for the benchmarks rollup.
|
||||
|
||||
Runs once per request rather than beside the daily transactions, which are built
|
||||
per entity type and would count every turn six times over. Independent of
|
||||
``disable_spend_logs``, because the rollup is what the benchmarks dashboard reads
|
||||
now. Never raises, and never blocks: a full queue drops the turn rather than
|
||||
applying backpressure to spend tracking.
|
||||
"""
|
||||
try:
|
||||
metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
|
||||
if not metadata.get("routing_decision"):
|
||||
return
|
||||
usage_raw: Final = metadata.get("usage_object")
|
||||
usage_obj: Final = usage_raw if isinstance(usage_raw, dict) else None
|
||||
cache_read_tokens: Final = _extract_cache_read_tokens(usage_obj) if usage_obj is not None else 0
|
||||
savings: Final = compute_savings_spend(
|
||||
model=payload.get("model", None),
|
||||
custom_llm_provider=payload.get("custom_llm_provider", None),
|
||||
compression_saved_tokens=extract_compression_saved_tokens(metadata),
|
||||
cache_read_input_tokens=cache_read_tokens,
|
||||
routing_decision=metadata.get("routing_decision"),
|
||||
model_id=payload.get("model_id"),
|
||||
llm_router=_get_llm_router,
|
||||
usage_object=usage_obj,
|
||||
cost_breakdown=metadata.get("cost_breakdown"),
|
||||
)
|
||||
turn: Final = build_turn_facts(
|
||||
payload=payload,
|
||||
metadata=metadata,
|
||||
autorouter_savings=savings.autorouter,
|
||||
cache_read_tokens=cache_read_tokens,
|
||||
cache_creation_tokens=_extract_cache_creation_tokens(usage_obj) if usage_obj is not None else 0,
|
||||
)
|
||||
if turn is not None:
|
||||
self.auto_router_session_queue.update_queue.put_nowait(turn)
|
||||
except Exception as e: # noqa: BLE001 # a dashboard rollup must never fail spend tracking
|
||||
verbose_proxy_logger.debug("auto_router_sessions: could not stage turn (%s)", e)
|
||||
|
||||
async def update_database(
|
||||
# LiteLLM management object fields
|
||||
|
|
@ -209,6 +250,8 @@ class DBSpendUpdateWriter:
|
|||
"disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur."
|
||||
)
|
||||
|
||||
self._enqueue_auto_router_turn(payload=payload)
|
||||
|
||||
# Single task replaces 11 create_task() calls
|
||||
asyncio.create_task(
|
||||
self._batch_database_updates(
|
||||
|
|
@ -809,6 +852,9 @@ class DBSpendUpdateWriter:
|
|||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
################## Auto-Router Benchmarks Rollup ##################
|
||||
await self.auto_router_session_queue.flush(prisma_client=prisma_client)
|
||||
|
||||
async def _commit_spend_updates_to_db_with_redis(
|
||||
self,
|
||||
prisma_client: PrismaClient,
|
||||
|
|
|
|||
|
|
@ -186,6 +186,20 @@ class SpendLogCleanup:
|
|||
time_column="start_time",
|
||||
)
|
||||
|
||||
async def _delete_old_auto_router_sessions(self, prisma_client: PrismaClient, cutoff_date: datetime) -> int:
|
||||
"""Expire auto-router rollups on the spend-log cutoff.
|
||||
|
||||
Keyed on last activity rather than session start, so a conversation still running
|
||||
when the cutoff passes is not pruned out from under itself.
|
||||
"""
|
||||
return await self._delete_old_rows_batched(
|
||||
prisma_client,
|
||||
cutoff_date,
|
||||
table_name="LiteLLM_AutoRouterSession",
|
||||
key_columns=("api_key", "session_id", "model_group"),
|
||||
time_column="last_turn_at",
|
||||
)
|
||||
|
||||
async def cleanup_old_spend_logs(self, prisma_client: PrismaClient) -> None:
|
||||
"""
|
||||
Main cleanup function. Deletes old spend logs in batches.
|
||||
|
|
@ -245,6 +259,9 @@ class SpendLogCleanup:
|
|||
index_deleted: Final = await self._delete_old_tool_index_rows(prisma_client, cutoff_date)
|
||||
verbose_proxy_logger.info("Deleted %s expired tool index rows", index_deleted)
|
||||
|
||||
sessions_deleted: Final = await self._delete_old_auto_router_sessions(prisma_client, cutoff_date)
|
||||
verbose_proxy_logger.info("Deleted %s expired auto-router session rollups", sessions_deleted)
|
||||
|
||||
except Exception as e:
|
||||
# .exception() captures the traceback; str(e) alone on a Prisma/DB
|
||||
# timeout is often empty and gives operators no signal to diagnose.
|
||||
|
|
|
|||
|
|
@ -16,10 +16,11 @@ import time
|
|||
import traceback
|
||||
import warnings
|
||||
from collections.abc import AsyncGenerator, Callable, Mapping
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from types import UnionType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Annotated,
|
||||
Any,
|
||||
Final,
|
||||
Literal,
|
||||
|
|
@ -530,6 +531,10 @@ from litellm.proxy.response_api_endpoints.endpoints import router as response_ro
|
|||
from litellm.proxy.route_llm_request import route_request
|
||||
from litellm.proxy.search_endpoints.endpoints import router as search_router
|
||||
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
|
||||
from litellm.proxy.spend_tracking.auto_router_benchmarks import (
|
||||
AutoRouterBenchmarksResponse,
|
||||
fetch_benchmarks,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start
|
||||
from litellm.proxy.spend_tracking.spend_management_endpoints import (
|
||||
router as spend_management_router,
|
||||
|
|
@ -16477,6 +16482,50 @@ async def get_adaptive_router_state(
|
|||
return {"routers": snapshots}
|
||||
|
||||
|
||||
def _auto_router_error(message: str) -> dict[str, str]:
|
||||
return {"error": message} # mutable-ok: HTTPException serializes its detail from a real dict
|
||||
|
||||
|
||||
@router.get(
|
||||
"/auto_router/benchmarks",
|
||||
tags=["auto_router"], # mutable-ok: FastAPI types tags as List[str]
|
||||
response_model=AutoRouterBenchmarksResponse,
|
||||
)
|
||||
async def get_auto_router_benchmarks(
|
||||
start_date: date,
|
||||
end_date: date,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
model_group: str | None = None,
|
||||
):
|
||||
"""Savings, session shape and prompt-cache behaviour for every auto-router.
|
||||
|
||||
Admin-only. Reads the per-session rollup only; no per-request table is scanned.
|
||||
`start_date` and `end_date` are inclusive calendar dates, clamped to the most recent
|
||||
30 days. Pass `model_group` to scope every figure to one auto-router.
|
||||
"""
|
||||
if not _user_has_admin_view(user_api_key_dict):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=_auto_router_error(CommonProxyErrors.not_allowed_access.value),
|
||||
)
|
||||
if end_date < start_date:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=_auto_router_error("end_date must not be earlier than start_date."),
|
||||
)
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail=_auto_router_error(CommonProxyErrors.db_not_connected_error.value),
|
||||
)
|
||||
return await fetch_benchmarks(
|
||||
prisma_client=prisma_client,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
model_group=model_group,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/routes", dependencies=[Depends(user_api_key_auth)])
|
||||
async def get_routes():
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1391,6 +1391,41 @@ model LiteLLM_AdaptiveRouterSession {
|
|||
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterSession {
|
||||
api_key String
|
||||
session_id String
|
||||
model_group String
|
||||
router_kind String
|
||||
baseline_model String?
|
||||
|
||||
turns Int @default(0)
|
||||
turns_with_usage Int @default(0)
|
||||
total_tokens BigInt @default(0)
|
||||
spend Float @default(0)
|
||||
baseline_spend Float @default(0)
|
||||
|
||||
first_visit_turns Int @default(0)
|
||||
first_visit_hits Int @default(0)
|
||||
warm_turns Int @default(0)
|
||||
warm_hits Int @default(0)
|
||||
expired_turns Int @default(0)
|
||||
expired_hits Int @default(0)
|
||||
unordered_turns Int @default(0)
|
||||
unordered_hits Int @default(0)
|
||||
|
||||
ephemeral_1h_turns Int @default(0)
|
||||
|
||||
tiers Json @default("{}")
|
||||
|
||||
first_turn_at DateTime
|
||||
last_turn_at DateTime
|
||||
updated_at DateTime @default(now()) @updatedAt
|
||||
|
||||
@@id([api_key, session_id, model_group])
|
||||
@@index([first_turn_at], map: "idx_auto_router_session_started")
|
||||
@@index([last_turn_at], map: "idx_auto_router_session_activity")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workflow Run Tracking
|
||||
//
|
||||
|
|
|
|||
258
litellm/proxy/spend_tracking/auto_router_benchmarks.py
Normal file
258
litellm/proxy/spend_tracking/auto_router_benchmarks.py
Normal file
|
|
@ -0,0 +1,258 @@
|
|||
"""Read path for the auto-router benchmarks dashboard.
|
||||
|
||||
Sums pre-folded rows from ``LiteLLM_AutoRouterSession`` and nothing else. ``summarize``
|
||||
produces both the per-router view and the totals, since averaging per-router percentages is
|
||||
wrong. Every miss has one of four causes on one denominator: cold, prefix changed, aged out, or a turn whose cache state could not be established.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import astuple, dataclass, fields
|
||||
from datetime import date, datetime, time, timedelta, timezone
|
||||
from functools import reduce
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from litellm.proxy.spend_tracking.auto_router_sessions import CACHE_TTL_1H_SECONDS, CACHE_TTL_5M_SECONDS
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
MAX_WINDOW_DAYS: Final = 30
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Counters:
|
||||
sessions: int = 0
|
||||
turns: int = 0
|
||||
turns_with_usage: int = 0
|
||||
total_tokens: int = 0
|
||||
total_session_seconds: float = 0.0
|
||||
spend: float = 0.0
|
||||
baseline_spend: float = 0.0
|
||||
first_visit_turns: int = 0
|
||||
first_visit_hits: int = 0
|
||||
warm_turns: int = 0
|
||||
warm_hits: int = 0
|
||||
expired_turns: int = 0
|
||||
expired_hits: int = 0
|
||||
unordered_turns: int = 0
|
||||
unordered_hits: int = 0
|
||||
ephemeral_1h_turns: int = 0
|
||||
|
||||
|
||||
class AutoRouterCacheBenchmark(BaseModel):
|
||||
ttl_seconds: float
|
||||
turns: int
|
||||
hits: int
|
||||
misses: int
|
||||
hit_rate_pct: float
|
||||
coverage_pct: float
|
||||
first_visit_turns: int
|
||||
first_visit_hits: int
|
||||
first_visit_hit_rate_pct: float
|
||||
warm_turns: int
|
||||
warm_hits: int
|
||||
warm_hit_rate_pct: float
|
||||
expired_turns: int
|
||||
expired_hits: int
|
||||
expired_hit_rate_pct: float
|
||||
unordered_turns: int
|
||||
unordered_hits: int
|
||||
cold_misses: int
|
||||
prefix_change_misses: int
|
||||
expired_misses: int
|
||||
unattributed_misses: int
|
||||
cold_miss_pct: float
|
||||
prefix_change_miss_pct: float
|
||||
expired_miss_pct: float
|
||||
unattributed_miss_pct: float
|
||||
|
||||
|
||||
class AutoRouterBenchmark(BaseModel):
|
||||
sessions: int
|
||||
turns: int
|
||||
total_tokens: int
|
||||
spend: float
|
||||
baseline_spend: float
|
||||
savings: float
|
||||
savings_pct: float
|
||||
saved_per_session: float
|
||||
avg_turns_per_session: float
|
||||
avg_session_seconds: float
|
||||
avg_tokens_per_session: float
|
||||
cache: AutoRouterCacheBenchmark | None
|
||||
|
||||
|
||||
class AutoRouterGroupBenchmark(BaseModel):
|
||||
model_group: str
|
||||
router_kind: str
|
||||
baseline_model: str | None
|
||||
benchmark: AutoRouterBenchmark
|
||||
|
||||
|
||||
class AutoRouterBenchmarksResponse(BaseModel):
|
||||
start_date: date
|
||||
end_date: date
|
||||
routers_in_scope: int
|
||||
totals: AutoRouterBenchmark
|
||||
groups: tuple[AutoRouterGroupBenchmark, ...]
|
||||
|
||||
|
||||
_DERIVED_COLUMNS: Final = ("sessions", "total_session_seconds")
|
||||
_SUM_COLUMNS: Final = tuple(field.name for field in fields(_Counters) if field.name not in _DERIVED_COLUMNS)
|
||||
|
||||
_AGGREGATE_SQL: Final = f"""
|
||||
SELECT
|
||||
model_group,
|
||||
MAX(router_kind) AS router_kind,
|
||||
MAX(baseline_model) AS baseline_model,
|
||||
COUNT(*) AS sessions,
|
||||
COALESCE(SUM(EXTRACT(EPOCH FROM (last_turn_at - first_turn_at))), 0) AS total_session_seconds,
|
||||
{", ".join(f"COALESCE(SUM({column}), 0) AS {column}" for column in _SUM_COLUMNS)}
|
||||
FROM "LiteLLM_AutoRouterSession"
|
||||
WHERE first_turn_at >= $1::timestamptz AT TIME ZONE \'UTC\'
|
||||
AND first_turn_at < $2::timestamptz AT TIME ZONE \'UTC\'
|
||||
AND ($3::text IS NULL OR model_group = $3::text)
|
||||
GROUP BY model_group
|
||||
ORDER BY SUM(spend) DESC
|
||||
"""
|
||||
|
||||
|
||||
def _pct(numerator: float, denominator: float) -> float:
|
||||
return 100.0 * numerator / denominator if denominator else 0.0
|
||||
|
||||
|
||||
def _per(numerator: float, denominator: float) -> float:
|
||||
return numerator / denominator if denominator else 0.0
|
||||
|
||||
|
||||
def _number(value: object) -> float:
|
||||
return float(value) if isinstance(value, (int, float)) else 0.0
|
||||
|
||||
|
||||
def _counters_from_row(row: Mapping[str, object]) -> _Counters:
|
||||
return _Counters(*(_number(row.get(field.name)) for field in fields(_Counters)))
|
||||
|
||||
|
||||
def _combine(left: _Counters, right: _Counters) -> _Counters:
|
||||
return _Counters(*(a + b for a, b in zip(astuple(left), astuple(right))))
|
||||
|
||||
|
||||
def _summarize_cache(counters: _Counters) -> AutoRouterCacheBenchmark | None:
|
||||
"""The cache view, or ``None`` when no turn in scope reported cache behaviour."""
|
||||
if counters.turns_with_usage == 0:
|
||||
return None
|
||||
covered: Final = counters.turns_with_usage
|
||||
hits: Final = counters.first_visit_hits + counters.warm_hits + counters.expired_hits + counters.unordered_hits
|
||||
misses: Final = covered - hits
|
||||
cold: Final = counters.first_visit_turns - counters.first_visit_hits
|
||||
prefix_changed: Final = counters.warm_turns - counters.warm_hits
|
||||
savable: Final = counters.expired_turns - counters.expired_hits
|
||||
unattributed: Final = counters.unordered_turns - counters.unordered_hits
|
||||
return AutoRouterCacheBenchmark(
|
||||
ttl_seconds=(
|
||||
CACHE_TTL_1H_SECONDS if counters.ephemeral_1h_turns * 2 > counters.turns else CACHE_TTL_5M_SECONDS
|
||||
),
|
||||
turns=covered,
|
||||
hits=hits,
|
||||
misses=misses,
|
||||
hit_rate_pct=_pct(hits, covered),
|
||||
coverage_pct=_pct(counters.turns_with_usage, counters.turns),
|
||||
first_visit_turns=counters.first_visit_turns,
|
||||
first_visit_hits=counters.first_visit_hits,
|
||||
first_visit_hit_rate_pct=_pct(counters.first_visit_hits, counters.first_visit_turns),
|
||||
warm_turns=counters.warm_turns,
|
||||
warm_hits=counters.warm_hits,
|
||||
warm_hit_rate_pct=_pct(counters.warm_hits, counters.warm_turns),
|
||||
expired_turns=counters.expired_turns,
|
||||
expired_hits=counters.expired_hits,
|
||||
expired_hit_rate_pct=_pct(counters.expired_hits, counters.expired_turns),
|
||||
unordered_turns=counters.unordered_turns,
|
||||
unordered_hits=counters.unordered_hits,
|
||||
cold_misses=cold,
|
||||
prefix_change_misses=prefix_changed,
|
||||
expired_misses=savable,
|
||||
unattributed_misses=unattributed,
|
||||
cold_miss_pct=_pct(cold, misses),
|
||||
prefix_change_miss_pct=_pct(prefix_changed, misses),
|
||||
expired_miss_pct=_pct(savable, misses),
|
||||
unattributed_miss_pct=_pct(unattributed, misses),
|
||||
)
|
||||
|
||||
|
||||
def summarize(counters: _Counters) -> AutoRouterBenchmark:
|
||||
"""One benchmark from raw counters, used for a single router and for the totals alike."""
|
||||
savings: Final = counters.baseline_spend - counters.spend
|
||||
return AutoRouterBenchmark(
|
||||
sessions=counters.sessions,
|
||||
turns=counters.turns,
|
||||
total_tokens=counters.total_tokens,
|
||||
spend=counters.spend,
|
||||
baseline_spend=counters.baseline_spend,
|
||||
savings=savings,
|
||||
savings_pct=_pct(savings, counters.baseline_spend),
|
||||
saved_per_session=_per(savings, counters.sessions),
|
||||
avg_turns_per_session=_per(counters.turns, counters.sessions),
|
||||
avg_session_seconds=_per(counters.total_session_seconds, counters.sessions),
|
||||
avg_tokens_per_session=_per(counters.total_tokens, counters.sessions),
|
||||
cache=_summarize_cache(counters),
|
||||
)
|
||||
|
||||
|
||||
def clamp_window(start_date: date, end_date: date) -> tuple[datetime, datetime]:
|
||||
"""The half-open UTC interval to read, clamped to ``MAX_WINDOW_DAYS``; ``end_date`` is
|
||||
inclusive to the caller, so the upper bound is the start of the following day."""
|
||||
span_start: Final = max(start_date, end_date - timedelta(days=MAX_WINDOW_DAYS - 1))
|
||||
return (
|
||||
datetime.combine(span_start, time.min, tzinfo=timezone.utc),
|
||||
datetime.combine(end_date + timedelta(days=1), time.min, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
|
||||
def build_response(
|
||||
rows: Sequence[Mapping[str, object]],
|
||||
start_date: date,
|
||||
end_date: date,
|
||||
) -> AutoRouterBenchmarksResponse:
|
||||
per_group: Final = tuple((row, _counters_from_row(row)) for row in rows)
|
||||
totals: Final = reduce(_combine, (counters for _, counters in per_group), _Counters())
|
||||
return AutoRouterBenchmarksResponse(
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
routers_in_scope=len(per_group),
|
||||
totals=summarize(totals),
|
||||
groups=tuple(
|
||||
AutoRouterGroupBenchmark(
|
||||
model_group=str(row.get("model_group") or ""),
|
||||
router_kind=str(row.get("router_kind") or ""),
|
||||
baseline_model=row.get("baseline_model") if isinstance(row.get("baseline_model"), str) else None,
|
||||
benchmark=summarize(counters),
|
||||
)
|
||||
for row, counters in per_group
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def fetch_benchmarks(
|
||||
prisma_client: PrismaClient,
|
||||
start_date: date,
|
||||
end_date: date,
|
||||
model_group: str | None = None,
|
||||
) -> AutoRouterBenchmarksResponse:
|
||||
"""Benchmarks for the window actually read; sessions are attributed to the window they
|
||||
started in, and the response echoes the clamped dates rather than the requested ones."""
|
||||
window_start, window_end = clamp_window(start_date, end_date)
|
||||
rows: Final = await prisma_client.db.query_raw(
|
||||
_AGGREGATE_SQL,
|
||||
window_start.isoformat(),
|
||||
window_end.isoformat(),
|
||||
model_group,
|
||||
)
|
||||
return build_response(
|
||||
rows=rows,
|
||||
start_date=window_start.date(),
|
||||
end_date=window_end.date() - timedelta(days=1),
|
||||
)
|
||||
273
litellm/proxy/spend_tracking/auto_router_sessions.py
Normal file
273
litellm/proxy/spend_tracking/auto_router_sessions.py
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
"""Per-(api_key, session, auto-router) rollup behind the auto-router benchmarks dashboard.
|
||||
|
||||
A turn enters the cache view only when it touched the cache, so a model with caching off
|
||||
contributes nothing to the buckets or the hit rate. ``tiers`` holds
|
||||
``{model: [refreshed_at, ttl, prefix_tokens]}``, so a turn's bucket is
|
||||
a question about one model's own record and the upsert answers it against the row it is
|
||||
already writing: absent or nothing live is a first visit, an earlier start is unordered, and
|
||||
otherwise warm or expired on the idle gap against the TTL the entry was written with.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.db.db_transaction_queue.base_update_queue import BaseUpdateQueue
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
CACHE_TTL_5M_SECONDS: Final = 300.0
|
||||
CACHE_TTL_1H_SECONDS: Final = 3600.0
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TurnFacts:
|
||||
"""One auto-routed turn, priced and reduced to what the rollup needs."""
|
||||
|
||||
api_key: str
|
||||
session_id: str
|
||||
model_group: str
|
||||
router_kind: str
|
||||
baseline_model: str | None
|
||||
model: str
|
||||
started_at: float
|
||||
total_tokens: int
|
||||
spend: float
|
||||
baseline_spend: float
|
||||
cache_hit: bool
|
||||
cache_creation_tokens: int
|
||||
cached_prefix_tokens: int
|
||||
ttl_seconds: float
|
||||
|
||||
|
||||
PARAM_NAMES: Final = (
|
||||
"api_key",
|
||||
"session_id",
|
||||
"model_group",
|
||||
"router_kind",
|
||||
"baseline_model",
|
||||
"model",
|
||||
"started_at",
|
||||
"total_tokens",
|
||||
"spend",
|
||||
"baseline_spend",
|
||||
"cache_hit",
|
||||
"written_tokens",
|
||||
"prefix_tokens",
|
||||
"one_hour",
|
||||
"ttl",
|
||||
)
|
||||
|
||||
(
|
||||
_API_KEY,
|
||||
_SESSION_ID,
|
||||
_MODEL_GROUP,
|
||||
_ROUTER_KIND,
|
||||
_BASELINE_MODEL,
|
||||
_MODEL,
|
||||
_STARTED_AT,
|
||||
_TOTAL_TOKENS,
|
||||
_SPEND,
|
||||
_BASELINE_SPEND,
|
||||
_CACHE_HIT,
|
||||
_WRITTEN_TOKENS,
|
||||
_PREFIX_TOKENS,
|
||||
_ONE_HOUR,
|
||||
_TTL,
|
||||
) = (f"${position}" for position in range(1, len(PARAM_NAMES) + 1))
|
||||
|
||||
|
||||
def bind(turn: TurnFacts) -> tuple[object, ...]:
|
||||
"""This turn's values, in ``PARAM_NAMES`` order."""
|
||||
return (
|
||||
turn.api_key,
|
||||
turn.session_id,
|
||||
turn.model_group,
|
||||
turn.router_kind,
|
||||
turn.baseline_model,
|
||||
turn.model,
|
||||
turn.started_at,
|
||||
turn.total_tokens,
|
||||
turn.spend,
|
||||
turn.baseline_spend,
|
||||
int(turn.cache_hit),
|
||||
turn.cache_creation_tokens,
|
||||
turn.cached_prefix_tokens,
|
||||
int(turn.ttl_seconds >= CACHE_TTL_1H_SECONDS),
|
||||
turn.ttl_seconds,
|
||||
)
|
||||
|
||||
|
||||
_SEEN: Final = f"t.tiers ? {_MODEL}"
|
||||
_CACHED_AT: Final = f"(t.tiers -> {_MODEL} ->> 0)::float8"
|
||||
_CACHED_TTL: Final = f"GREATEST((t.tiers -> {_MODEL} ->> 1)::float8, 1)"
|
||||
_CACHED_TOKENS: Final = f"(t.tiers -> {_MODEL} ->> 2)::float8"
|
||||
_IDLE: Final = f"({_STARTED_AT}::float8 - {_CACHED_AT})"
|
||||
|
||||
_COVERED: Final = f"{_PREFIX_TOKENS}::float8 > 0"
|
||||
_LIVE: Final = f"{_SEEN} AND {_CACHED_TOKENS} > 0"
|
||||
_UNORDERED: Final = f"{_COVERED} AND {_LIVE} AND {_IDLE} < 0"
|
||||
_WARM: Final = f"{_COVERED} AND {_LIVE} AND {_IDLE} >= 0 AND {_IDLE} <= {_CACHED_TTL}"
|
||||
_EXPIRED: Final = f"{_COVERED} AND {_LIVE} AND {_IDLE} > {_CACHED_TTL}"
|
||||
_REFRESHED: Final = f"NOT ({_SEEN}) OR ({_PREFIX_TOKENS}::float8 > 0 AND {_IDLE} >= 0)"
|
||||
_REWROTE: Final = f"NOT ({_SEEN}) OR ({_WRITTEN_TOKENS}::float8 > 0 AND {_IDLE} >= 0)"
|
||||
|
||||
_UPSERT_SQL: Final = f"""
|
||||
INSERT INTO "LiteLLM_AutoRouterSession" AS t (
|
||||
api_key, session_id, model_group, router_kind, baseline_model,
|
||||
turns, turns_with_usage, total_tokens, spend, baseline_spend,
|
||||
first_visit_turns, first_visit_hits, ephemeral_1h_turns,
|
||||
tiers, first_turn_at, last_turn_at, updated_at
|
||||
)
|
||||
VALUES (
|
||||
{_API_KEY}, {_SESSION_ID}, {_MODEL_GROUP}, {_ROUTER_KIND}, {_BASELINE_MODEL},
|
||||
1, CASE WHEN {_COVERED} THEN 1 ELSE 0 END, {_TOTAL_TOKENS}::bigint, {_SPEND}, {_BASELINE_SPEND},
|
||||
CASE WHEN {_COVERED} THEN 1 ELSE 0 END,
|
||||
CASE WHEN {_COVERED} THEN {_CACHE_HIT} ELSE 0 END,
|
||||
{_ONE_HOUR},
|
||||
jsonb_build_object({_MODEL}, jsonb_build_array(
|
||||
{_STARTED_AT}::float8, {_TTL}::float8, {_PREFIX_TOKENS}::float8
|
||||
)),
|
||||
to_timestamp({_STARTED_AT}::float8) AT TIME ZONE 'UTC',
|
||||
to_timestamp({_STARTED_AT}::float8) AT TIME ZONE 'UTC',
|
||||
NOW()
|
||||
)
|
||||
ON CONFLICT (api_key, session_id, model_group) DO UPDATE SET
|
||||
turns = t.turns + 1,
|
||||
turns_with_usage = t.turns_with_usage + CASE WHEN {_COVERED} THEN 1 ELSE 0 END,
|
||||
total_tokens = t.total_tokens + {_TOTAL_TOKENS}::bigint,
|
||||
spend = t.spend + {_SPEND},
|
||||
baseline_spend = t.baseline_spend + {_BASELINE_SPEND},
|
||||
|
||||
first_visit_turns = t.first_visit_turns + CASE WHEN {_COVERED} AND NOT ({_LIVE}) THEN 1 ELSE 0 END,
|
||||
first_visit_hits = t.first_visit_hits + CASE WHEN {_COVERED} AND NOT ({_LIVE}) THEN {_CACHE_HIT} ELSE 0 END,
|
||||
unordered_turns = t.unordered_turns + CASE WHEN {_UNORDERED} THEN 1 ELSE 0 END,
|
||||
unordered_hits = t.unordered_hits + CASE WHEN {_UNORDERED} THEN {_CACHE_HIT} ELSE 0 END,
|
||||
warm_turns = t.warm_turns + CASE WHEN {_WARM} THEN 1 ELSE 0 END,
|
||||
warm_hits = t.warm_hits + CASE WHEN {_WARM} THEN {_CACHE_HIT} ELSE 0 END,
|
||||
expired_turns = t.expired_turns + CASE WHEN {_EXPIRED} THEN 1 ELSE 0 END,
|
||||
expired_hits = t.expired_hits + CASE WHEN {_EXPIRED} THEN {_CACHE_HIT} ELSE 0 END,
|
||||
|
||||
|
||||
ephemeral_1h_turns = t.ephemeral_1h_turns + {_ONE_HOUR},
|
||||
baseline_model = COALESCE(t.baseline_model, {_BASELINE_MODEL}),
|
||||
tiers = t.tiers || jsonb_build_object({_MODEL}, jsonb_build_array(
|
||||
CASE WHEN {_REFRESHED} THEN {_STARTED_AT}::float8 ELSE {_CACHED_AT} END,
|
||||
CASE WHEN {_REWROTE} THEN {_TTL}::float8 ELSE {_CACHED_TTL} END,
|
||||
CASE WHEN {_REFRESHED} THEN {_PREFIX_TOKENS}::float8 ELSE {_CACHED_TOKENS} END
|
||||
)),
|
||||
first_turn_at = LEAST(t.first_turn_at, to_timestamp({_STARTED_AT}::float8) AT TIME ZONE 'UTC'),
|
||||
last_turn_at = GREATEST(t.last_turn_at, to_timestamp({_STARTED_AT}::float8) AT TIME ZONE 'UTC'),
|
||||
updated_at = NOW()
|
||||
"""
|
||||
|
||||
|
||||
class AutoRouterSessionQueue(BaseUpdateQueue):
|
||||
"""Stages turns in memory; writes an interval as one ordered batch on the spend flush.
|
||||
|
||||
Batched because a busy interval drains up to ``MAX_IN_MEMORY_QUEUE_FLUSH_COUNT`` turns.
|
||||
Ordered because a session's classification depends on the turn before it. Never raises.
|
||||
"""
|
||||
|
||||
async def flush(self, prisma_client: PrismaClient) -> None:
|
||||
staged: Final[Sequence[TurnFacts]] = await self.flush_all_updates_from_in_memory_queue()
|
||||
if not staged:
|
||||
return
|
||||
try:
|
||||
async with prisma_client.db.batch_() as batcher:
|
||||
for turn in sorted(staged, key=lambda staged_turn: staged_turn.started_at):
|
||||
batcher.execute_raw(_UPSERT_SQL, *bind(turn))
|
||||
except Exception as e: # noqa: BLE001 # a dashboard rollup must never fail spend tracking
|
||||
verbose_proxy_logger.warning("auto_router_sessions: dropped %d turns (%s)", len(staged), e)
|
||||
|
||||
|
||||
def ttl_seconds(usage_obj: Mapping[str, object] | None) -> float:
|
||||
"""The cache tier this turn wrote against; no one-hour evidence means the 5m default."""
|
||||
if usage_obj is None:
|
||||
return CACHE_TTL_5M_SECONDS
|
||||
details: Final = usage_obj.get("cache_creation_token_details")
|
||||
if not isinstance(details, Mapping):
|
||||
return CACHE_TTL_5M_SECONDS
|
||||
if int(details.get("ephemeral_1h_input_tokens") or 0) > 0:
|
||||
return CACHE_TTL_1H_SECONDS
|
||||
return CACHE_TTL_5M_SECONDS
|
||||
|
||||
|
||||
def _as_utc(moment: datetime) -> float:
|
||||
return (moment.replace(tzinfo=timezone.utc) if moment.tzinfo is None else moment).timestamp()
|
||||
|
||||
|
||||
def _started_at(start_time: object) -> float | None:
|
||||
if isinstance(start_time, datetime):
|
||||
return _as_utc(start_time)
|
||||
if isinstance(start_time, str):
|
||||
try:
|
||||
return _as_utc(datetime.fromisoformat(start_time))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _str_or_none(value: object) -> str | None:
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def build_turn_facts(
|
||||
payload: Mapping[str, object],
|
||||
metadata: Mapping[str, object],
|
||||
autorouter_savings: float,
|
||||
cache_read_tokens: int,
|
||||
cache_creation_tokens: int,
|
||||
) -> TurnFacts | None:
|
||||
"""One spend-log payload as a rollup turn, or ``None`` if it was not auto-routed.
|
||||
|
||||
A recorded ``routing_decision`` is what says the request was auto-routed, and names the kind.
|
||||
"""
|
||||
decision: Final = metadata.get("routing_decision")
|
||||
if not isinstance(decision, Mapping):
|
||||
return None
|
||||
router_kind: Final = decision.get("router_type")
|
||||
api_key: Final = payload.get("api_key")
|
||||
session_id: Final = payload.get("session_id")
|
||||
model_group: Final = payload.get("model_group")
|
||||
model: Final = payload.get("model")
|
||||
if not (
|
||||
isinstance(router_kind, str)
|
||||
and isinstance(api_key, str)
|
||||
and isinstance(session_id, str)
|
||||
and isinstance(model_group, str)
|
||||
and isinstance(model, str)
|
||||
and api_key
|
||||
and session_id
|
||||
and model_group
|
||||
and model
|
||||
):
|
||||
return None
|
||||
started_at: Final = _started_at(payload.get("startTime"))
|
||||
if started_at is None:
|
||||
return None
|
||||
usage_raw: Final = metadata.get("usage_object")
|
||||
usage_obj: Final = usage_raw if isinstance(usage_raw, Mapping) else None
|
||||
spend: Final = float(payload.get("spend") or 0.0)
|
||||
return TurnFacts(
|
||||
api_key=api_key,
|
||||
session_id=session_id,
|
||||
model_group=model_group,
|
||||
router_kind=router_kind,
|
||||
baseline_model=litellm.autorouter_savings_baseline_model,
|
||||
model=model,
|
||||
started_at=started_at,
|
||||
total_tokens=int(payload.get("prompt_tokens") or 0) + int(payload.get("completion_tokens") or 0),
|
||||
spend=spend,
|
||||
baseline_spend=spend + autorouter_savings,
|
||||
cache_hit=cache_read_tokens > 0,
|
||||
cache_creation_tokens=cache_creation_tokens,
|
||||
cached_prefix_tokens=max(cache_read_tokens, 0) + max(cache_creation_tokens, 0),
|
||||
ttl_seconds=ttl_seconds(usage_obj),
|
||||
)
|
||||
|
|
@ -1391,6 +1391,41 @@ model LiteLLM_AdaptiveRouterSession {
|
|||
@@index([last_activity_at], map: "idx_adaptive_router_session_activity")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterSession {
|
||||
api_key String
|
||||
session_id String
|
||||
model_group String
|
||||
router_kind String
|
||||
baseline_model String?
|
||||
|
||||
turns Int @default(0)
|
||||
turns_with_usage Int @default(0)
|
||||
total_tokens BigInt @default(0)
|
||||
spend Float @default(0)
|
||||
baseline_spend Float @default(0)
|
||||
|
||||
first_visit_turns Int @default(0)
|
||||
first_visit_hits Int @default(0)
|
||||
warm_turns Int @default(0)
|
||||
warm_hits Int @default(0)
|
||||
expired_turns Int @default(0)
|
||||
expired_hits Int @default(0)
|
||||
unordered_turns Int @default(0)
|
||||
unordered_hits Int @default(0)
|
||||
|
||||
ephemeral_1h_turns Int @default(0)
|
||||
|
||||
tiers Json @default("{}")
|
||||
|
||||
first_turn_at DateTime
|
||||
last_turn_at DateTime
|
||||
updated_at DateTime @default(now()) @updatedAt
|
||||
|
||||
@@id([api_key, session_id, model_group])
|
||||
@@index([first_turn_at], map: "idx_auto_router_session_started")
|
||||
@@index([last_turn_at], map: "idx_auto_router_session_activity")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workflow Run Tracking
|
||||
//
|
||||
|
|
|
|||
0
tests/proxy_behavior/spend/__init__.py
Normal file
0
tests/proxy_behavior/spend/__init__.py
Normal file
36
tests/proxy_behavior/spend/conftest.py
Normal file
36
tests/proxy_behavior/spend/conftest.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
"""A real Postgres connection for the rollup's SQL.
|
||||
|
||||
The auto-router rollup classifies each turn inside its upsert, against the row's own
|
||||
stored state, so the classification only exists when a real database evaluates it.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest_asyncio
|
||||
from prisma import Prisma
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PrismaClientShim:
|
||||
"""What the rollup writer needs from litellm's PrismaClient: a connected `db`."""
|
||||
|
||||
db: Prisma
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope="session", loop_scope="session")
|
||||
async def prisma_db():
|
||||
db = Prisma()
|
||||
await db.connect()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
await db.disconnect()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(loop_scope="session")
|
||||
async def rollup_client(prisma_db):
|
||||
await prisma_db.execute_raw('DELETE FROM "LiteLLM_AutoRouterSession"')
|
||||
try:
|
||||
yield PrismaClientShim(db=prisma_db)
|
||||
finally:
|
||||
await prisma_db.execute_raw('DELETE FROM "LiteLLM_AutoRouterSession"')
|
||||
237
tests/proxy_behavior/spend/test_auto_router_session_rollup.py
Normal file
237
tests/proxy_behavior/spend/test_auto_router_session_rollup.py
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
"""How the auto-router rollup buckets a turn, evaluated by a real Postgres.
|
||||
|
||||
The upsert classifies each turn against the session's own cache record, so these assertions
|
||||
are about SQL. They cover what the dashboard depends on: the three buckets partition every
|
||||
turn, a tier that aged out is told from one that is still warm, the warming estimate is
|
||||
priced on the prefix that was actually cached, and one caller cannot write into another's
|
||||
rollup by reusing a session id.
|
||||
"""
|
||||
|
||||
import datetime as dt
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.spend_tracking.auto_router_sessions import AutoRouterSessionQueue, TurnFacts
|
||||
|
||||
pytestmark = pytest.mark.asyncio(loop_scope="session")
|
||||
|
||||
HAIKU = "anthropic/claude-haiku-4-5"
|
||||
OPUS = "anthropic/claude-opus-4-8"
|
||||
T0 = dt.datetime(2026, 8, 3, 12, 0, tzinfo=dt.timezone.utc).timestamp()
|
||||
|
||||
TURN = TurnFacts(
|
||||
api_key="key-a",
|
||||
session_id="sess-1",
|
||||
model_group="claude-auto",
|
||||
router_kind="complexity",
|
||||
baseline_model=OPUS,
|
||||
model=HAIKU,
|
||||
started_at=T0,
|
||||
total_tokens=1000,
|
||||
spend=0.01,
|
||||
baseline_spend=0.05,
|
||||
cache_hit=False,
|
||||
cache_creation_tokens=2000,
|
||||
cached_prefix_tokens=2000,
|
||||
ttl_seconds=300.0,
|
||||
)
|
||||
|
||||
|
||||
async def _flush(client, turns) -> None:
|
||||
queue = AutoRouterSessionQueue()
|
||||
for turn in turns:
|
||||
await queue.update_queue.put(turn)
|
||||
await queue.flush(prisma_client=client)
|
||||
|
||||
|
||||
async def _rows(db) -> list[dict]:
|
||||
return await db.query_raw('SELECT * FROM "LiteLLM_AutoRouterSession" ORDER BY api_key, session_id')
|
||||
|
||||
|
||||
def _epoch_of(stored: str) -> float:
|
||||
parsed = dt.datetime.fromisoformat(stored)
|
||||
return (parsed.replace(tzinfo=dt.timezone.utc) if parsed.tzinfo is None else parsed).timestamp()
|
||||
|
||||
|
||||
async def test_a_sessions_opening_turn_on_a_tier_is_a_first_visit(rollup_client, prisma_db):
|
||||
await _flush(rollup_client, [TURN])
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert (row["turns"], row["first_visit_turns"], row["warm_turns"], row["expired_turns"]) == (1, 1, 0, 0)
|
||||
assert row["tiers"] == {HAIKU: [T0, 300.0, 2000]}
|
||||
|
||||
|
||||
async def test_a_second_tier_is_its_own_first_visit(rollup_client, prisma_db):
|
||||
await _flush(rollup_client, [TURN, replace(TURN, model=OPUS, started_at=T0 + 10)])
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert (row["turns"], row["first_visit_turns"]) == (2, 2)
|
||||
assert set(row["tiers"]) == {HAIKU, OPUS}
|
||||
|
||||
|
||||
async def test_a_tier_used_again_inside_its_ttl_is_warm(rollup_client, prisma_db):
|
||||
await _flush(rollup_client, [TURN, replace(TURN, started_at=T0 + 60, cache_hit=True)])
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert (row["warm_turns"], row["warm_hits"], row["expired_turns"]) == (1, 1, 0)
|
||||
|
||||
|
||||
async def test_a_tier_used_again_past_its_ttl_has_expired(rollup_client, prisma_db):
|
||||
await _flush(rollup_client, [TURN, replace(TURN, started_at=T0 + 900)])
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert (row["warm_turns"], row["expired_turns"], row["expired_hits"]) == (0, 1, 0)
|
||||
|
||||
|
||||
async def test_expiry_measures_against_the_ttl_the_cache_was_written_with(rollup_client, prisma_db):
|
||||
"""A one-hour entry is still warm at 30 minutes, even though the turn reading it
|
||||
reports no one-hour evidence of its own and so carries the five minute default."""
|
||||
await _flush(
|
||||
rollup_client,
|
||||
[
|
||||
replace(TURN, ttl_seconds=3600.0, cache_creation_tokens=2000),
|
||||
replace(TURN, started_at=T0 + 1800, ttl_seconds=300.0, cache_hit=True, cache_creation_tokens=0),
|
||||
],
|
||||
)
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert (row["warm_turns"], row["expired_turns"]) == (1, 0)
|
||||
|
||||
|
||||
async def test_the_three_buckets_partition_every_turn(rollup_client, prisma_db):
|
||||
await _flush(
|
||||
rollup_client,
|
||||
[
|
||||
TURN,
|
||||
replace(TURN, started_at=T0 + 10),
|
||||
replace(TURN, model=OPUS, started_at=T0 + 20),
|
||||
replace(TURN, started_at=T0 + 5000),
|
||||
],
|
||||
)
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert row["first_visit_turns"] + row["warm_turns"] + row["expired_turns"] == row["turns"] == 4
|
||||
|
||||
|
||||
async def test_a_turn_that_only_read_the_cache_leaves_the_written_terms_alone(rollup_client, prisma_db):
|
||||
await _flush(
|
||||
rollup_client,
|
||||
[
|
||||
replace(TURN, ttl_seconds=3600.0, cache_creation_tokens=5000, cached_prefix_tokens=5000),
|
||||
replace(TURN, started_at=T0 + 10, cache_hit=True, cache_creation_tokens=0, cached_prefix_tokens=5000),
|
||||
],
|
||||
)
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert row["tiers"][HAIKU] == [T0 + 10, 3600.0, 5000]
|
||||
|
||||
|
||||
async def test_counters_accumulate_across_flushes(rollup_client, prisma_db):
|
||||
await _flush(rollup_client, [TURN])
|
||||
await _flush(rollup_client, [replace(TURN, started_at=T0 + 10)])
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert row["turns"] == 2
|
||||
assert row["spend"] == pytest.approx(0.02)
|
||||
assert row["baseline_spend"] == pytest.approx(0.10)
|
||||
assert row["total_tokens"] == 2000
|
||||
assert _epoch_of(row["first_turn_at"]) == pytest.approx(T0, abs=0.001)
|
||||
assert _epoch_of(row["last_turn_at"]) == pytest.approx(T0 + 10, abs=0.001)
|
||||
|
||||
|
||||
async def test_a_late_turn_cannot_rewind_the_session(rollup_client, prisma_db):
|
||||
await _flush(rollup_client, [replace(TURN, started_at=T0 + 600)])
|
||||
await _flush(rollup_client, [replace(TURN, started_at=T0)])
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert row["turns"] == 2
|
||||
assert row["tiers"][HAIKU][0] == T0 + 600
|
||||
assert _epoch_of(row["last_turn_at"]) == pytest.approx(T0 + 600, abs=0.001)
|
||||
assert _epoch_of(row["first_turn_at"]) == pytest.approx(T0, abs=0.001)
|
||||
|
||||
|
||||
async def test_two_callers_reusing_one_session_id_keep_separate_rollups(rollup_client, prisma_db):
|
||||
await _flush(
|
||||
rollup_client,
|
||||
[
|
||||
replace(TURN, api_key="key-a", model=HAIKU),
|
||||
replace(TURN, api_key="key-b", model=OPUS, started_at=T0 + 10),
|
||||
],
|
||||
)
|
||||
rows = await _rows(prisma_db)
|
||||
assert [row["api_key"] for row in rows] == ["key-a", "key-b"]
|
||||
assert all(row["turns"] == 1 and row["first_visit_turns"] == 1 for row in rows)
|
||||
|
||||
|
||||
async def test_a_turn_arriving_before_an_already_recorded_one_is_not_called_warm(rollup_client, prisma_db):
|
||||
"""Its cache state at its own time is unknowable, so it abstains rather than being
|
||||
guessed at; a negative idle gap is not evidence of warmth."""
|
||||
await _flush(rollup_client, [replace(TURN, started_at=T0 + 600)])
|
||||
await _flush(rollup_client, [replace(TURN, started_at=T0)])
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert row["turns"] == 2
|
||||
assert row["unordered_turns"] == 1
|
||||
assert row["warm_turns"] == 0
|
||||
assert row["expired_turns"] == 0
|
||||
assert row["first_visit_turns"] == 1
|
||||
assert row["tiers"][HAIKU][0] == T0 + 600
|
||||
|
||||
|
||||
async def test_every_turn_lands_in_exactly_one_of_the_four_buckets(rollup_client, prisma_db):
|
||||
await _flush(
|
||||
rollup_client,
|
||||
[
|
||||
TURN,
|
||||
replace(TURN, started_at=T0 + 10),
|
||||
replace(TURN, model=OPUS, started_at=T0 + 20),
|
||||
replace(TURN, started_at=T0 + 5000),
|
||||
],
|
||||
)
|
||||
await _flush(rollup_client, [replace(TURN, started_at=T0 + 5)])
|
||||
row = (await _rows(prisma_db))[0]
|
||||
buckets = row["first_visit_turns"] + row["warm_turns"] + row["expired_turns"] + row["unordered_turns"]
|
||||
assert buckets == row["turns"] == 5
|
||||
assert row["unordered_turns"] == 1
|
||||
|
||||
|
||||
async def test_a_first_visit_to_a_new_tier_keeps_its_own_ttl_even_with_no_cache_write(rollup_client, prisma_db):
|
||||
await _flush(
|
||||
rollup_client,
|
||||
[
|
||||
TURN,
|
||||
replace(
|
||||
TURN,
|
||||
model=OPUS,
|
||||
started_at=T0 + 10,
|
||||
ttl_seconds=3600.0,
|
||||
cache_creation_tokens=0,
|
||||
cached_prefix_tokens=0,
|
||||
),
|
||||
],
|
||||
)
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert row["tiers"][OPUS] == [T0 + 10, 3600.0, 0]
|
||||
|
||||
|
||||
async def test_a_growing_conversation_records_the_whole_live_prefix(rollup_client, prisma_db):
|
||||
"""A warm turn on a growing prompt writes only the new segment, so recording
|
||||
cache_creation_tokens alone would shrink the prefix and under-price later replays.
|
||||
The live prefix is what was read plus what was written."""
|
||||
await _flush(
|
||||
rollup_client,
|
||||
[
|
||||
replace(TURN, cache_creation_tokens=2000, cached_prefix_tokens=2000),
|
||||
replace(TURN, started_at=T0 + 60, cache_hit=True, cache_creation_tokens=500, cached_prefix_tokens=2500),
|
||||
],
|
||||
)
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert row["warm_turns"] == 1
|
||||
assert row["tiers"][HAIKU] == [T0 + 60, 300.0, 2500]
|
||||
|
||||
|
||||
async def test_a_turn_that_touched_no_cache_is_left_out_of_the_cache_view(rollup_client, prisma_db):
|
||||
"""A model with caching off would otherwise read as a wall of first visits and drag the
|
||||
hit rate down; it counts as a turn and nothing else."""
|
||||
await _flush(
|
||||
rollup_client,
|
||||
[
|
||||
replace(TURN, cache_creation_tokens=0, cached_prefix_tokens=0),
|
||||
replace(TURN, started_at=T0 + 10, cache_creation_tokens=0, cached_prefix_tokens=0),
|
||||
],
|
||||
)
|
||||
row = (await _rows(prisma_db))[0]
|
||||
assert row["turns"] == 2
|
||||
assert row["turns_with_usage"] == 0
|
||||
assert row["first_visit_turns"] + row["warm_turns"] + row["expired_turns"] + row["unordered_turns"] == 0
|
||||
|
|
@ -0,0 +1,246 @@
|
|||
"""Read-path derivations for the auto-router benchmarks dashboard.
|
||||
|
||||
Every figure the dashboard shows is a pure function of the rollup counters, so all of it
|
||||
is exercised here without a database.
|
||||
"""
|
||||
|
||||
import datetime as dt
|
||||
from dataclasses import fields
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.spend_tracking.auto_router_benchmarks import (
|
||||
MAX_WINDOW_DAYS,
|
||||
_Counters,
|
||||
build_response,
|
||||
clamp_window,
|
||||
summarize,
|
||||
)
|
||||
|
||||
|
||||
def _row(model_group: str, **overrides: object) -> dict[str, object]:
|
||||
"""Every counter at zero unless the case under test says otherwise."""
|
||||
return {
|
||||
"model_group": model_group,
|
||||
"router_kind": "complexity",
|
||||
"baseline_model": "anthropic/claude-opus-4-8",
|
||||
**{field.name: 0 for field in fields(_Counters)},
|
||||
"sessions": 1,
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
class TestSummarize:
|
||||
def test_savings_are_the_difference_between_the_two_arms(self):
|
||||
result = summarize(_Counters(sessions=4, turns=40, spend=364.59, baseline_spend=414.63))
|
||||
assert result.savings == pytest.approx(50.04)
|
||||
assert result.savings_pct == pytest.approx(100 * 50.04 / 414.63)
|
||||
assert result.saved_per_session == pytest.approx(50.04 / 4)
|
||||
assert result.avg_turns_per_session == pytest.approx(10.0)
|
||||
|
||||
def test_a_cache_thrashing_router_reports_a_signed_loss(self):
|
||||
result = summarize(_Counters(sessions=1, turns=1, spend=5.0, baseline_spend=4.0))
|
||||
assert result.savings == pytest.approx(-1.0)
|
||||
assert result.savings_pct < 0
|
||||
|
||||
def test_session_shape_averages_over_sessions_not_turns(self):
|
||||
result = summarize(
|
||||
_Counters(sessions=2, turns=64, total_tokens=10_000, total_session_seconds=7200.0)
|
||||
)
|
||||
assert result.avg_turns_per_session == pytest.approx(32.0)
|
||||
assert result.avg_session_seconds == pytest.approx(3600.0)
|
||||
assert result.avg_tokens_per_session == pytest.approx(5000.0)
|
||||
|
||||
def test_an_empty_window_divides_by_nothing(self):
|
||||
result = summarize(_Counters())
|
||||
assert result.sessions == 0
|
||||
assert result.savings_pct == 0.0
|
||||
assert result.saved_per_session == 0.0
|
||||
assert result.avg_turns_per_session == 0.0
|
||||
assert result.cache is None
|
||||
|
||||
|
||||
class TestCacheView:
|
||||
def test_hit_rate_is_weighted_by_turn_count_not_averaged_across_buckets(self):
|
||||
cache = summarize(
|
||||
_Counters(
|
||||
sessions=1,
|
||||
turns=3145,
|
||||
turns_with_usage=3145,
|
||||
warm_turns=2560,
|
||||
warm_hits=2491,
|
||||
first_visit_turns=146,
|
||||
first_visit_hits=15,
|
||||
expired_turns=439,
|
||||
expired_hits=348,
|
||||
)
|
||||
).cache
|
||||
assert cache is not None
|
||||
hits = 2491 + 15 + 348
|
||||
assert cache.hit_rate_pct == pytest.approx(100 * hits / 3145)
|
||||
mean_of_bucket_rates = (
|
||||
cache.warm_hit_rate_pct + cache.first_visit_hit_rate_pct + cache.expired_hit_rate_pct
|
||||
) / 3
|
||||
assert cache.hit_rate_pct != pytest.approx(mean_of_bucket_rates)
|
||||
assert cache.hit_rate_pct > mean_of_bucket_rates
|
||||
|
||||
def test_traffic_that_never_touched_the_cache_is_left_out_of_the_hit_rate(self):
|
||||
"""A model with caching off would otherwise read as a wall of misses. It is absent
|
||||
from the buckets, and coverage says how much of the traffic that was."""
|
||||
cache = summarize(
|
||||
_Counters(sessions=1, turns=100, turns_with_usage=40, warm_turns=40, warm_hits=36)
|
||||
).cache
|
||||
assert cache is not None
|
||||
assert cache.turns == 40
|
||||
assert cache.coverage_pct == pytest.approx(40.0)
|
||||
assert cache.hit_rate_pct == pytest.approx(90.0)
|
||||
assert cache.misses == 4
|
||||
|
||||
def test_the_three_buckets_partition_every_turn(self):
|
||||
counters = _Counters(
|
||||
sessions=1, turns=10, turns_with_usage=10, warm_turns=6, first_visit_turns=2, expired_turns=2
|
||||
)
|
||||
cache = summarize(counters).cache
|
||||
assert cache is not None
|
||||
assert cache.warm_turns + cache.first_visit_turns + cache.expired_turns == cache.turns
|
||||
|
||||
def test_every_miss_has_one_cause_and_they_stack_to_the_whole(self):
|
||||
"""A miss is cold by design, a changed prefix, an expiry, or a turn whose cache
|
||||
state could not be established."""
|
||||
cache = summarize(
|
||||
_Counters(
|
||||
sessions=1,
|
||||
turns=24,
|
||||
turns_with_usage=24,
|
||||
first_visit_turns=5,
|
||||
first_visit_hits=1,
|
||||
warm_turns=10,
|
||||
warm_hits=8,
|
||||
expired_turns=5,
|
||||
expired_hits=2,
|
||||
unordered_turns=4,
|
||||
unordered_hits=3,
|
||||
)
|
||||
).cache
|
||||
assert cache is not None
|
||||
assert cache.hits == 14
|
||||
assert cache.misses == 10
|
||||
assert (cache.cold_misses, cache.prefix_change_misses, cache.expired_misses) == (4, 2, 3)
|
||||
assert cache.unattributed_misses == 1
|
||||
assert (
|
||||
cache.cold_misses + cache.prefix_change_misses + cache.expired_misses + cache.unattributed_misses
|
||||
== cache.misses
|
||||
)
|
||||
assert (
|
||||
cache.cold_miss_pct
|
||||
+ cache.prefix_change_miss_pct
|
||||
+ cache.expired_miss_pct
|
||||
+ cache.unattributed_miss_pct
|
||||
== pytest.approx(100.0)
|
||||
)
|
||||
|
||||
def test_an_unordered_turn_still_counts_toward_the_headline_hit_rate(self):
|
||||
"""Its cause is unknowable, but the provider still said whether it hit, so the
|
||||
rate a reader looks at stays exact and only the attribution abstains."""
|
||||
cache = summarize(
|
||||
_Counters(sessions=1, turns=10, turns_with_usage=10, warm_turns=6, warm_hits=6, unordered_turns=4, unordered_hits=2)
|
||||
).cache
|
||||
assert cache is not None
|
||||
assert cache.hits == 8
|
||||
assert cache.hit_rate_pct == pytest.approx(80.0)
|
||||
|
||||
def test_a_router_with_no_cache_evidence_has_no_cache_view(self):
|
||||
assert summarize(_Counters(sessions=1, turns=5, turns_with_usage=0)).cache is None
|
||||
|
||||
def test_ttl_follows_the_tier_the_majority_of_turns_used(self):
|
||||
five_minute = summarize(_Counters(sessions=1, turns=10, turns_with_usage=10, ephemeral_1h_turns=4)).cache
|
||||
one_hour = summarize(_Counters(sessions=1, turns=10, turns_with_usage=10, ephemeral_1h_turns=6)).cache
|
||||
assert five_minute is not None and one_hour is not None
|
||||
assert five_minute.ttl_seconds == 300.0
|
||||
assert one_hour.ttl_seconds == 3600.0
|
||||
|
||||
|
||||
class TestTotals:
|
||||
def test_totals_sum_the_counters_rather_than_averaging_group_rates(self):
|
||||
"""A big cheap router and a small expensive one must not be weighted equally."""
|
||||
response = build_response(
|
||||
rows=[
|
||||
_row(
|
||||
"big",
|
||||
turns=1000,
|
||||
turns_with_usage=1000,
|
||||
warm_turns=1000,
|
||||
warm_hits=900,
|
||||
first_visit_turns=0,
|
||||
first_visit_hits=0,
|
||||
expired_turns=0,
|
||||
expired_hits=0,
|
||||
),
|
||||
_row(
|
||||
"small",
|
||||
turns=10,
|
||||
turns_with_usage=10,
|
||||
warm_turns=10,
|
||||
warm_hits=1,
|
||||
first_visit_turns=0,
|
||||
first_visit_hits=0,
|
||||
expired_turns=0,
|
||||
expired_hits=0,
|
||||
),
|
||||
],
|
||||
start_date=dt.date(2026, 7, 5),
|
||||
end_date=dt.date(2026, 8, 3),
|
||||
)
|
||||
assert response.routers_in_scope == 2
|
||||
assert response.totals.cache is not None
|
||||
assert response.totals.cache.hit_rate_pct == pytest.approx(100 * 901 / 1010)
|
||||
group_rates = [g.benchmark.cache.hit_rate_pct for g in response.groups if g.benchmark.cache]
|
||||
assert response.totals.cache.hit_rate_pct != pytest.approx(sum(group_rates) / len(group_rates))
|
||||
|
||||
def test_totals_dollars_are_the_sum_of_every_router(self):
|
||||
response = build_response(
|
||||
rows=[_row("a", spend=3.0, baseline_spend=5.0), _row("b", spend=1.0, baseline_spend=9.0)],
|
||||
start_date=dt.date(2026, 7, 5),
|
||||
end_date=dt.date(2026, 8, 3),
|
||||
)
|
||||
assert response.totals.spend == pytest.approx(4.0)
|
||||
assert response.totals.baseline_spend == pytest.approx(14.0)
|
||||
assert response.totals.savings == pytest.approx(10.0)
|
||||
assert response.totals.sessions == 2
|
||||
|
||||
def test_the_response_echoes_the_window_actually_read(self):
|
||||
"""A caller asking for a year is told it got a month, not handed month-sized
|
||||
numbers under year-sized dates."""
|
||||
response = build_response(rows=[], start_date=dt.date(2026, 7, 5), end_date=dt.date(2026, 8, 3))
|
||||
assert (response.start_date, response.end_date) == (dt.date(2026, 7, 5), dt.date(2026, 8, 3))
|
||||
|
||||
def test_an_empty_window_still_answers_with_zeroed_totals(self):
|
||||
response = build_response(rows=[], start_date=dt.date(2026, 8, 1), end_date=dt.date(2026, 8, 3))
|
||||
assert response.routers_in_scope == 0
|
||||
assert response.groups == ()
|
||||
assert response.totals.turns == 0
|
||||
|
||||
def test_group_identity_is_carried_through(self):
|
||||
response = build_response(
|
||||
rows=[_row("claude-auto")], start_date=dt.date(2026, 8, 1), end_date=dt.date(2026, 8, 3)
|
||||
)
|
||||
assert response.groups[0].model_group == "claude-auto"
|
||||
assert response.groups[0].router_kind == "complexity"
|
||||
assert response.groups[0].baseline_model == "anthropic/claude-opus-4-8"
|
||||
|
||||
|
||||
class TestWindow:
|
||||
def test_end_date_is_inclusive(self):
|
||||
start, end = clamp_window(dt.date(2026, 8, 3), dt.date(2026, 8, 3))
|
||||
assert start == dt.datetime(2026, 8, 3, tzinfo=dt.timezone.utc)
|
||||
assert end == dt.datetime(2026, 8, 4, tzinfo=dt.timezone.utc)
|
||||
|
||||
def test_a_wider_request_is_clamped_to_the_cap_measured_in_dates_spanned(self):
|
||||
start, end = clamp_window(dt.date(2020, 1, 1), dt.date(2026, 8, 3))
|
||||
assert (end.date() - start.date()).days == MAX_WINDOW_DAYS
|
||||
assert start == dt.datetime(2026, 7, 5, tzinfo=dt.timezone.utc)
|
||||
|
||||
def test_a_window_inside_the_cap_is_left_alone(self):
|
||||
start, end = clamp_window(dt.date(2026, 8, 1), dt.date(2026, 8, 3))
|
||||
assert start == dt.datetime(2026, 8, 1, tzinfo=dt.timezone.utc)
|
||||
assert end == dt.datetime(2026, 8, 4, tzinfo=dt.timezone.utc)
|
||||
|
|
@ -0,0 +1,124 @@
|
|||
"""Reducing one spend-log payload to a rollup turn.
|
||||
|
||||
Bucketing a turn is done by the upsert against the session's own cache record, so it is
|
||||
covered in tests/proxy_behavior/spend against a real Postgres. What is pure, and covered
|
||||
here, is deciding whether a request is an auto-routed turn at all and what it contributes.
|
||||
"""
|
||||
|
||||
import datetime as dt
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.spend_tracking.auto_router_sessions import (
|
||||
CACHE_TTL_1H_SECONDS,
|
||||
CACHE_TTL_5M_SECONDS,
|
||||
build_turn_facts,
|
||||
ttl_seconds,
|
||||
)
|
||||
|
||||
T0 = dt.datetime(2026, 8, 3, 12, 0, tzinfo=dt.timezone.utc)
|
||||
|
||||
|
||||
def _payload(**overrides: object) -> dict[str, object]:
|
||||
base: dict[str, object] = {
|
||||
"api_key": "hashed-key",
|
||||
"session_id": "sess-1",
|
||||
"model_group": "claude-auto",
|
||||
"model": "claude-haiku-4-5",
|
||||
"custom_llm_provider": "anthropic",
|
||||
"startTime": T0.isoformat(),
|
||||
"spend": 0.25,
|
||||
"prompt_tokens": 900,
|
||||
"completion_tokens": 100,
|
||||
}
|
||||
return {**base, **overrides}
|
||||
|
||||
|
||||
def _metadata(**overrides: object) -> dict[str, object]:
|
||||
base: dict[str, object] = {
|
||||
"routing_decision": {"router_type": "complexity"},
|
||||
"usage_object": {"cache_read_input_tokens": 500, "cache_creation_input_tokens": 0},
|
||||
}
|
||||
return {**base, **overrides}
|
||||
|
||||
|
||||
def _build(payload=None, metadata=None, **kwargs):
|
||||
return build_turn_facts(
|
||||
payload=payload if payload is not None else _payload(),
|
||||
metadata=metadata if metadata is not None else _metadata(),
|
||||
autorouter_savings=kwargs.get("autorouter_savings", 0.75),
|
||||
cache_read_tokens=kwargs.get("cache_read_tokens", 500),
|
||||
cache_creation_tokens=kwargs.get("cache_creation_tokens", 0),
|
||||
)
|
||||
|
||||
|
||||
class TestNotAutoRouted:
|
||||
def test_a_request_with_no_routing_decision_is_not_a_turn(self):
|
||||
assert _build(metadata={"usage_object": {}}) is None
|
||||
|
||||
def test_a_routing_decision_without_a_kind_is_not_a_turn(self):
|
||||
assert _build(metadata=_metadata(routing_decision={})) is None
|
||||
|
||||
@pytest.mark.parametrize("field", ["api_key", "session_id", "model_group", "model"])
|
||||
def test_a_turn_missing_any_identity_field_is_dropped(self, field: str):
|
||||
assert _build(payload=_payload(**{field: ""})) is None
|
||||
assert _build(payload=_payload(**{field: None})) is None
|
||||
|
||||
def test_an_unparseable_start_time_is_dropped(self):
|
||||
assert _build(payload=_payload(startTime="not-a-timestamp")) is None
|
||||
|
||||
|
||||
class TestTurnFacts:
|
||||
def test_the_router_kind_is_read_from_the_decision_the_router_recorded(self):
|
||||
built = _build()
|
||||
assert built is not None and built.router_kind == "complexity"
|
||||
|
||||
def test_the_baseline_arm_is_this_turn_plus_what_the_router_saved(self):
|
||||
built = _build(autorouter_savings=0.75)
|
||||
assert built is not None
|
||||
assert (built.spend, built.baseline_spend) == (pytest.approx(0.25), pytest.approx(1.0))
|
||||
|
||||
def test_a_route_that_lost_money_carries_a_baseline_below_what_was_paid(self):
|
||||
built = _build(autorouter_savings=-0.10)
|
||||
assert built is not None and built.baseline_spend == pytest.approx(0.15)
|
||||
|
||||
def test_tokens_are_the_whole_turn(self):
|
||||
built = _build()
|
||||
assert built is not None and built.total_tokens == 1000
|
||||
|
||||
def test_a_naive_start_time_is_read_as_utc(self):
|
||||
naive = _build(payload=_payload(startTime=T0.replace(tzinfo=None)))
|
||||
aware = _build(payload=_payload(startTime=T0))
|
||||
assert naive is not None and aware is not None
|
||||
assert naive.started_at == aware.started_at == T0.timestamp()
|
||||
|
||||
def test_a_cache_read_is_a_hit(self):
|
||||
built = _build(cache_read_tokens=1)
|
||||
assert built is not None and built.cache_hit is True
|
||||
|
||||
def test_no_cache_read_is_a_miss(self):
|
||||
built = _build(cache_read_tokens=0)
|
||||
assert built is not None and built.cache_hit is False
|
||||
|
||||
|
||||
class TestCacheEvidence:
|
||||
def test_the_live_prefix_is_what_was_read_plus_what_was_written(self):
|
||||
built = _build(cache_read_tokens=500, cache_creation_tokens=200)
|
||||
assert built is not None and built.cached_prefix_tokens == 700
|
||||
|
||||
def test_a_turn_that_touched_no_cache_has_no_prefix(self):
|
||||
"""Coverage keys off this: a model with caching off is absent from the cache view
|
||||
rather than counted as a miss."""
|
||||
built = _build(cache_read_tokens=0, cache_creation_tokens=0)
|
||||
assert built is not None and built.cached_prefix_tokens == 0
|
||||
|
||||
def test_a_turn_with_no_one_hour_evidence_is_scored_against_the_five_minute_tier(self):
|
||||
assert ttl_seconds({"cache_creation_input_tokens": 10}) == CACHE_TTL_5M_SECONDS
|
||||
assert ttl_seconds(None) == CACHE_TTL_5M_SECONDS
|
||||
|
||||
def test_one_hour_cache_writes_are_scored_against_the_one_hour_tier(self):
|
||||
usage = {
|
||||
"cache_creation_input_tokens": 10,
|
||||
"cache_creation_token_details": {"ephemeral_1h_input_tokens": 10},
|
||||
}
|
||||
assert ttl_seconds(usage) == CACHE_TTL_1H_SECONDS
|
||||
|
|
@ -158,8 +158,8 @@ async def test_cleanup_old_spend_logs_batch_deletion():
|
|||
mock_db = MagicMock()
|
||||
|
||||
# Mock execute_raw to return deleted counts (3 spend-log batches, then the
|
||||
# tool-index cleanup's first batch returning 0)
|
||||
mock_db.execute_raw = AsyncMock(side_effect=[1000, 500, 0, 0])
|
||||
# tool-index and auto-router-session cleanups each returning 0 on their first batch)
|
||||
mock_db.execute_raw = AsyncMock(side_effect=[1000, 500, 0, 0, 0])
|
||||
|
||||
# Wire up mocks
|
||||
mock_prisma_client.db = mock_db
|
||||
|
|
@ -179,7 +179,7 @@ async def test_cleanup_old_spend_logs_batch_deletion():
|
|||
await cleaner.cleanup_old_spend_logs(mock_prisma_client)
|
||||
|
||||
# Validate batching and deletion via raw SQL
|
||||
assert mock_db.execute_raw.call_count == 4
|
||||
assert mock_db.execute_raw.call_count == 5
|
||||
|
||||
# Check the first call argument
|
||||
call_args_sql = mock_db.execute_raw.call_args_list[0][0][0]
|
||||
|
|
@ -193,6 +193,12 @@ async def test_cleanup_old_spend_logs_batch_deletion():
|
|||
tool_index_sql = mock_db.execute_raw.call_args_list[3][0][0]
|
||||
assert 'DELETE FROM "LiteLLM_SpendLogToolIndex"' in tool_index_sql
|
||||
|
||||
# The auto-router rollup expires on the same cutoff, keyed on last activity so a
|
||||
# conversation still running when the cutoff passes is not pruned mid-session
|
||||
session_sql = mock_db.execute_raw.call_args_list[4][0][0]
|
||||
assert 'DELETE FROM "LiteLLM_AutoRouterSession"' in session_sql
|
||||
assert '"last_turn_at" <' in session_sql
|
||||
|
||||
# The LiteLLM_DailyToolSpend rollup must outlive spend-log retention: it is
|
||||
# the only copy of tool spend history once its per-request sources expire,
|
||||
# so spend-log cleanup must never touch it.
|
||||
|
|
@ -285,7 +291,7 @@ async def test_cleanup_uses_delete_when_partitioning_not_enabled():
|
|||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
mock_prisma_client = MagicMock()
|
||||
mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[10, 0, 0])
|
||||
mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[10, 0, 0, 0])
|
||||
|
||||
partition_manager = MagicMock()
|
||||
partition_manager.is_partitioned = AsyncMock(return_value=True)
|
||||
|
|
@ -316,7 +322,7 @@ async def test_cleanup_uses_delete_when_not_partitioned():
|
|||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
mock_prisma_client = MagicMock()
|
||||
mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[10, 0, 0])
|
||||
mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[10, 0, 0, 0])
|
||||
|
||||
partition_manager = MagicMock()
|
||||
partition_manager.is_partitioned = AsyncMock(return_value=False)
|
||||
|
|
@ -335,7 +341,7 @@ async def test_cleanup_uses_delete_when_not_partitioned():
|
|||
await cleaner.cleanup_old_spend_logs(mock_prisma_client)
|
||||
|
||||
partition_manager.drop_partitions_older_than.assert_not_awaited()
|
||||
assert mock_prisma_client.db.execute_raw.await_count == 3
|
||||
assert mock_prisma_client.db.execute_raw.await_count == 4
|
||||
delete_sql = mock_prisma_client.db.execute_raw.call_args_list[0][0][0]
|
||||
assert 'DELETE FROM "LiteLLM_SpendLogs"' in delete_sql
|
||||
|
||||
|
|
|
|||
164
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
164
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -760,6 +760,30 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/auto_router/benchmarks": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* Get Auto Router Benchmarks
|
||||
* @description Savings, session shape and prompt-cache behaviour for every auto-router.
|
||||
*
|
||||
* Admin-only. Reads the per-session rollup only; no per-request table is scanned.
|
||||
* `start_date` and `end_date` are inclusive calendar dates, clamped to the most recent
|
||||
* 30 days. Pass `model_group` to scope every figure to one auto-router.
|
||||
*/
|
||||
get: operations["get_auto_router_benchmarks_auto_router_benchmarks_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/azure/{endpoint}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -21062,6 +21086,113 @@ export interface components {
|
|||
[key: string]: unknown;
|
||||
} | null;
|
||||
};
|
||||
/** AutoRouterBenchmark */
|
||||
AutoRouterBenchmark: {
|
||||
/** Avg Session Seconds */
|
||||
avg_session_seconds: number;
|
||||
/** Avg Tokens Per Session */
|
||||
avg_tokens_per_session: number;
|
||||
/** Avg Turns Per Session */
|
||||
avg_turns_per_session: number;
|
||||
/** Baseline Spend */
|
||||
baseline_spend: number;
|
||||
cache: components["schemas"]["AutoRouterCacheBenchmark"] | null;
|
||||
/** Saved Per Session */
|
||||
saved_per_session: number;
|
||||
/** Savings */
|
||||
savings: number;
|
||||
/** Savings Pct */
|
||||
savings_pct: number;
|
||||
/** Sessions */
|
||||
sessions: number;
|
||||
/** Spend */
|
||||
spend: number;
|
||||
/** Total Tokens */
|
||||
total_tokens: number;
|
||||
/** Turns */
|
||||
turns: number;
|
||||
};
|
||||
/** AutoRouterBenchmarksResponse */
|
||||
AutoRouterBenchmarksResponse: {
|
||||
/**
|
||||
* End Date
|
||||
* Format: date
|
||||
*/
|
||||
end_date: string;
|
||||
/** Groups */
|
||||
groups: components["schemas"]["AutoRouterGroupBenchmark"][];
|
||||
/** Routers In Scope */
|
||||
routers_in_scope: number;
|
||||
/**
|
||||
* Start Date
|
||||
* Format: date
|
||||
*/
|
||||
start_date: string;
|
||||
totals: components["schemas"]["AutoRouterBenchmark"];
|
||||
};
|
||||
/** AutoRouterCacheBenchmark */
|
||||
AutoRouterCacheBenchmark: {
|
||||
/** Cold Miss Pct */
|
||||
cold_miss_pct: number;
|
||||
/** Cold Misses */
|
||||
cold_misses: number;
|
||||
/** Coverage Pct */
|
||||
coverage_pct: number;
|
||||
/** Expired Hit Rate Pct */
|
||||
expired_hit_rate_pct: number;
|
||||
/** Expired Hits */
|
||||
expired_hits: number;
|
||||
/** Expired Miss Pct */
|
||||
expired_miss_pct: number;
|
||||
/** Expired Misses */
|
||||
expired_misses: number;
|
||||
/** Expired Turns */
|
||||
expired_turns: number;
|
||||
/** First Visit Hit Rate Pct */
|
||||
first_visit_hit_rate_pct: number;
|
||||
/** First Visit Hits */
|
||||
first_visit_hits: number;
|
||||
/** First Visit Turns */
|
||||
first_visit_turns: number;
|
||||
/** Hit Rate Pct */
|
||||
hit_rate_pct: number;
|
||||
/** Hits */
|
||||
hits: number;
|
||||
/** Misses */
|
||||
misses: number;
|
||||
/** Prefix Change Miss Pct */
|
||||
prefix_change_miss_pct: number;
|
||||
/** Prefix Change Misses */
|
||||
prefix_change_misses: number;
|
||||
/** Ttl Seconds */
|
||||
ttl_seconds: number;
|
||||
/** Turns */
|
||||
turns: number;
|
||||
/** Unattributed Miss Pct */
|
||||
unattributed_miss_pct: number;
|
||||
/** Unattributed Misses */
|
||||
unattributed_misses: number;
|
||||
/** Unordered Hits */
|
||||
unordered_hits: number;
|
||||
/** Unordered Turns */
|
||||
unordered_turns: number;
|
||||
/** Warm Hit Rate Pct */
|
||||
warm_hit_rate_pct: number;
|
||||
/** Warm Hits */
|
||||
warm_hits: number;
|
||||
/** Warm Turns */
|
||||
warm_turns: number;
|
||||
};
|
||||
/** AutoRouterGroupBenchmark */
|
||||
AutoRouterGroupBenchmark: {
|
||||
/** Baseline Model */
|
||||
baseline_model: string | null;
|
||||
benchmark: components["schemas"]["AutoRouterBenchmark"];
|
||||
/** Model Group */
|
||||
model_group: string;
|
||||
/** Router Kind */
|
||||
router_kind: string;
|
||||
};
|
||||
/** BaseLitellmParams */
|
||||
"BaseLitellmParams-Input": {
|
||||
/**
|
||||
|
|
@ -35698,6 +35829,39 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
get_auto_router_benchmarks_auto_router_benchmarks_get: {
|
||||
parameters: {
|
||||
query: {
|
||||
start_date: string;
|
||||
end_date: string;
|
||||
model_group?: string | null;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["AutoRouterBenchmarksResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
azure_proxy_route_azure__endpoint__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue