mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
feat(ui): add internal-user savings and auto-router usage
This commit is contained in:
Tin Chi Lo
parent
252a0f1eac
commit
0068df5a8b
23 changed files with 1321 additions and 229 deletions
|
|
@ -0,0 +1,42 @@
|
|||
CREATE TABLE IF NOT EXISTS "LiteLLM_AutoRouterUserSession" (
|
||||
"user_id" TEXT NOT NULL,
|
||||
"api_key" TEXT NOT NULL,
|
||||
"session_id" TEXT NOT NULL,
|
||||
"router_name" TEXT NOT NULL,
|
||||
"router_type" TEXT NOT NULL,
|
||||
"first_turn_at" TIMESTAMP(3) NOT NULL,
|
||||
"last_turn_at" TIMESTAMP(3) NOT NULL,
|
||||
"last_model" TEXT NOT NULL,
|
||||
"models" JSONB NOT NULL DEFAULT '{}',
|
||||
"turns" INTEGER NOT NULL DEFAULT 0,
|
||||
"unordered_turns" INTEGER NOT NULL DEFAULT 0,
|
||||
"covered_turns" INTEGER NOT NULL DEFAULT 0,
|
||||
"cache_hits" INTEGER NOT NULL DEFAULT 0,
|
||||
"same_model_turns" INTEGER NOT NULL DEFAULT 0,
|
||||
"same_model_hits" INTEGER NOT NULL DEFAULT 0,
|
||||
"first_visit_turns" INTEGER NOT NULL DEFAULT 0,
|
||||
"first_visit_hits" INTEGER NOT NULL DEFAULT 0,
|
||||
"return_turns" INTEGER NOT NULL DEFAULT 0,
|
||||
"return_hits" INTEGER NOT NULL DEFAULT 0,
|
||||
"return_expired_misses" INTEGER NOT NULL DEFAULT 0,
|
||||
"return_within_ttl_misses" INTEGER NOT NULL DEFAULT 0,
|
||||
"ttl_5m_turns" INTEGER NOT NULL DEFAULT 0,
|
||||
"ttl_1h_turns" INTEGER NOT NULL DEFAULT 0,
|
||||
"total_tokens" BIGINT NOT NULL DEFAULT 0,
|
||||
"spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
"saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
"savings_estimated_turns" INTEGER NOT NULL DEFAULT 0,
|
||||
"savings_estimated_actual_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
"savings_estimated_saved_spend" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
"savings_estimated_baseline_models" JSONB NOT NULL DEFAULT '{}',
|
||||
"classifier_cost" DOUBLE PRECISION NOT NULL DEFAULT 0,
|
||||
"classifier_cost_recorded_turns" INTEGER NOT NULL DEFAULT 0,
|
||||
"tier_turns" JSONB NOT NULL DEFAULT '{}',
|
||||
"baseline_models" JSONB NOT NULL DEFAULT '{}',
|
||||
|
||||
CONSTRAINT "LiteLLM_AutoRouterUserSession_pkey" PRIMARY KEY ("user_id", "api_key", "session_id", "router_name")
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "idx_autorouter_user_session_last_turn" ON "LiteLLM_AutoRouterUserSession"("last_turn_at");
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "idx_autorouter_user_session_user_last_turn" ON "LiteLLM_AutoRouterUserSession"("user_id", "last_turn_at");
|
||||
|
|
@ -1620,6 +1620,47 @@ model LiteLLM_AutoRouterSession {
|
|||
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterUserSession {
|
||||
user_id String
|
||||
api_key String
|
||||
session_id String
|
||||
router_name String
|
||||
router_type String
|
||||
first_turn_at DateTime
|
||||
last_turn_at DateTime
|
||||
last_model String
|
||||
models Json @default("{}")
|
||||
turns Int @default(0)
|
||||
unordered_turns Int @default(0)
|
||||
covered_turns Int @default(0)
|
||||
cache_hits Int @default(0)
|
||||
same_model_turns Int @default(0)
|
||||
same_model_hits Int @default(0)
|
||||
first_visit_turns Int @default(0)
|
||||
first_visit_hits Int @default(0)
|
||||
return_turns Int @default(0)
|
||||
return_hits Int @default(0)
|
||||
return_expired_misses Int @default(0)
|
||||
return_within_ttl_misses Int @default(0)
|
||||
ttl_5m_turns Int @default(0)
|
||||
ttl_1h_turns Int @default(0)
|
||||
total_tokens BigInt @default(0)
|
||||
spend Float @default(0)
|
||||
saved_spend Float @default(0)
|
||||
savings_estimated_turns Int @default(0)
|
||||
savings_estimated_actual_spend Float @default(0)
|
||||
savings_estimated_saved_spend Float @default(0)
|
||||
savings_estimated_baseline_models Json @default("{}")
|
||||
classifier_cost Float @default(0)
|
||||
classifier_cost_recorded_turns Int @default(0)
|
||||
tier_turns Json @default("{}")
|
||||
baseline_models Json @default("{}")
|
||||
|
||||
@@id([user_id, api_key, session_id, router_name])
|
||||
@@index([last_turn_at], map: "idx_autorouter_user_session_last_turn")
|
||||
@@index([user_id, last_turn_at], map: "idx_autorouter_user_session_user_last_turn")
|
||||
}
|
||||
|
||||
// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
|
||||
// either direction. forward duplicates the requests the keys did not route through the
|
||||
// router through it, answering whether they should adopt it; reverse duplicates the
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Per-session auto-router benchmarks rollup.
|
|||
At request time the spend writer builds one AutoRouterTurnTransaction per successful
|
||||
auto-routed request (a request whose metadata carries a routing_decision) and queues it
|
||||
on the prisma client. The spend-log flush job drains the queue into
|
||||
LiteLLM_AutoRouterSession with one conditional upsert per turn: the statement classifies
|
||||
key and user session rollups with one atomic statement per turn: each upsert classifies
|
||||
the turn (same model, first visit, return to a model the session already used, out of
|
||||
order) against the row's own columns, so nothing is read before the write and concurrent
|
||||
pods compose. The benchmarks endpoint aggregates these rows and never touches
|
||||
|
|
@ -35,10 +35,27 @@ if TYPE_CHECKING:
|
|||
CACHE_TTL_5M_SECONDS: Final = 300
|
||||
CACHE_TTL_1H_SECONDS: Final = 3600
|
||||
|
||||
AUTOROUTER_BENCHMARKS_SQL: Final = """
|
||||
_SESSION_COLUMNS: Final = """
|
||||
api_key, session_id, router_name, router_type, first_turn_at, last_turn_at,
|
||||
last_model, models, turns, unordered_turns, covered_turns, cache_hits,
|
||||
same_model_turns, same_model_hits, first_visit_turns, first_visit_hits,
|
||||
return_turns, return_hits, return_expired_misses, return_within_ttl_misses,
|
||||
ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend, classifier_cost, classifier_cost_recorded_turns, tier_turns,
|
||||
baseline_models, savings_estimated_turns, savings_estimated_actual_spend, savings_estimated_saved_spend,
|
||||
savings_estimated_baseline_models
|
||||
"""
|
||||
|
||||
AUTOROUTER_BENCHMARKS_SQL: Final = f"""
|
||||
WITH windowed AS (
|
||||
SELECT * FROM "LiteLLM_AutoRouterSession"
|
||||
WHERE last_turn_at >= $1::timestamp
|
||||
SELECT {_SESSION_COLUMNS} FROM "LiteLLM_AutoRouterSession"
|
||||
WHERE $4::text IS NULL
|
||||
AND last_turn_at >= $1::timestamp
|
||||
AND first_turn_at < $2::timestamp
|
||||
AND ($3::text IS NULL OR api_key = $3::text)
|
||||
UNION ALL
|
||||
SELECT {_SESSION_COLUMNS} FROM "LiteLLM_AutoRouterUserSession"
|
||||
WHERE (($4::text IS NOT NULL AND user_id = $4::text) OR ($4::text IS NULL AND api_key = ''))
|
||||
AND last_turn_at >= $1::timestamp
|
||||
AND first_turn_at < $2::timestamp
|
||||
AND ($3::text IS NULL OR api_key = $3::text)
|
||||
),
|
||||
|
|
@ -53,7 +70,7 @@ tier_maps AS (
|
|||
)
|
||||
SELECT
|
||||
agg.*,
|
||||
COALESCE(tier_maps.tier_turns, '{}'::jsonb) AS tier_turns
|
||||
COALESCE(tier_maps.tier_turns, '{{}}'::jsonb) AS tier_turns
|
||||
FROM (
|
||||
SELECT
|
||||
router_name,
|
||||
|
|
@ -111,6 +128,7 @@ class AutoRouterTurnTransaction:
|
|||
savings_estimated_turns: int = 0
|
||||
savings_estimated_actual_spend: float = 0.0
|
||||
savings_estimated_saved_spend: float = 0.0
|
||||
user_id: str = ""
|
||||
|
||||
|
||||
class TurnCacheFacts(NamedTuple):
|
||||
|
|
@ -214,10 +232,11 @@ def build_autorouter_turn_transaction(
|
|||
if not isinstance(routing_decision, Mapping) or not routing_decision:
|
||||
return None
|
||||
router_name: Final = routing_decision.get("router_model_name") or payload.get("model_group")
|
||||
api_key: Final = payload.get("api_key")
|
||||
api_key: Final = payload.get("api_key") or ""
|
||||
user_id: Final = payload.get("user") or ""
|
||||
session_id: Final = payload.get("session_id")
|
||||
model: Final = payload.get("model")
|
||||
if not (isinstance(router_name, str) and router_name and api_key and session_id and model):
|
||||
if not (isinstance(router_name, str) and router_name and (api_key or user_id) and session_id and model):
|
||||
return None
|
||||
turn_at: Final = _turn_time_utc(str(payload.get("startTime") or ""))
|
||||
if turn_at is None:
|
||||
|
|
@ -236,6 +255,7 @@ def build_autorouter_turn_transaction(
|
|||
estimated_savings: Final = recorded_estimated_autorouter_savings(metadata)
|
||||
return AutoRouterTurnTransaction(
|
||||
api_key=api_key,
|
||||
user_id=user_id,
|
||||
session_id=bounded_session_id(session_id),
|
||||
router_name=router_name,
|
||||
router_type=str(routing_decision.get("router_type") or "unknown"),
|
||||
|
|
@ -293,18 +313,18 @@ _RETURN_MISS: Final = (
|
|||
_IDLE_SECONDS: Final = f"EXTRACT(EPOCH FROM {_TURN_AT}::timestamp) - (t.models -> {_MODEL} ->> 'at')::float8"
|
||||
_CACHE_TOUCHED: Final = f"{_TOUCHED}::int = 1"
|
||||
|
||||
UPSERT_AUTOROUTER_SESSION_SQL: Final = f"""
|
||||
INSERT INTO "LiteLLM_AutoRouterSession" AS t (
|
||||
api_key, session_id, router_name, router_type, first_turn_at, last_turn_at,
|
||||
last_model, models, turns, unordered_turns, covered_turns, cache_hits,
|
||||
same_model_turns, same_model_hits, first_visit_turns, first_visit_hits,
|
||||
return_turns, return_hits, return_expired_misses, return_within_ttl_misses,
|
||||
ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend, classifier_cost, classifier_cost_recorded_turns, tier_turns,
|
||||
baseline_models, savings_estimated_turns, savings_estimated_actual_spend, savings_estimated_saved_spend,
|
||||
savings_estimated_baseline_models
|
||||
|
||||
def _session_upsert_sql(*, user_scoped: bool) -> str:
|
||||
table_name: Final = "LiteLLM_AutoRouterUserSession" if user_scoped else "LiteLLM_AutoRouterSession"
|
||||
user_column: Final = "user_id, " if user_scoped else ""
|
||||
user_value: Final = f"{_p('user_id')}::text, " if user_scoped else ""
|
||||
required_identity: Final = _p("user_id" if user_scoped else "api_key")
|
||||
return f"""
|
||||
INSERT INTO "{table_name}" AS t (
|
||||
{user_column}{_SESSION_COLUMNS}
|
||||
)
|
||||
VALUES (
|
||||
{_p("api_key")}, {_p("session_id")}, {_p("router_name")}, {_p("router_type")}, {_TURN_AT}::timestamp, {_TURN_AT}::timestamp,
|
||||
SELECT
|
||||
{user_value}{_p("api_key")}, {_p("session_id")}, {_p("router_name")}, {_p("router_type")}, {_TURN_AT}::timestamp, {_TURN_AT}::timestamp,
|
||||
{_MODEL}, jsonb_build_object({_MODEL}, jsonb_build_object('at', EXTRACT(EPOCH FROM {_TURN_AT}::timestamp), 'ttl', {_CACHE_TTL}::int)),
|
||||
1, 0, {_COVERED}::int, {_CACHE_HIT}::int,
|
||||
0, 0, 1, {_CACHE_HIT}::int,
|
||||
|
|
@ -315,8 +335,8 @@ VALUES (
|
|||
{_p("classifier_cost")}::float8, 1, {_TIER_DELTA}, {_BASELINE_DELTA},
|
||||
{_p("savings_estimated_turns")}::int, {_p("savings_estimated_actual_spend")}::float8,
|
||||
{_p("savings_estimated_saved_spend")}::float8, {_ESTIMATED_BASELINE_DELTA}
|
||||
)
|
||||
ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
|
||||
WHERE {required_identity}::text <> ''
|
||||
ON CONFLICT ({user_column}api_key, session_id, router_name) DO UPDATE SET
|
||||
turns = t.turns + 1,
|
||||
total_tokens = t.total_tokens + EXCLUDED.total_tokens,
|
||||
spend = t.spend + EXCLUDED.spend,
|
||||
|
|
@ -365,6 +385,17 @@ ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
|
|||
"""
|
||||
|
||||
|
||||
UPSERT_AUTOROUTER_SESSION_SQL: Final = f"""
|
||||
WITH key_rollup AS (
|
||||
{_session_upsert_sql(user_scoped=False)}
|
||||
RETURNING 1
|
||||
)
|
||||
{_session_upsert_sql(user_scoped=True)}
|
||||
"""
|
||||
|
||||
UPSERT_AUTOROUTER_USER_SESSION_SQL: Final = _session_upsert_sql(user_scoped=True)
|
||||
|
||||
|
||||
def _as_sql_param(value: str | float | bool | datetime | None) -> str | float | None:
|
||||
if isinstance(value, bool):
|
||||
return int(value)
|
||||
|
|
@ -377,18 +408,23 @@ def _upsert_params(transaction: AutoRouterTurnTransaction) -> tuple[str | float
|
|||
return tuple(_as_sql_param(getattr(transaction, name)) for name in _UPSERT_PARAM_FIELDS)
|
||||
|
||||
|
||||
async def write_autorouter_turn(db: SupportsExecuteRaw, transaction: AutoRouterTurnTransaction) -> None:
|
||||
await db.execute_raw(UPSERT_AUTOROUTER_SESSION_SQL, *_upsert_params(transaction))
|
||||
async def write_autorouter_turn(
|
||||
db: SupportsExecuteRaw,
|
||||
transaction: AutoRouterTurnTransaction,
|
||||
statement: str = UPSERT_AUTOROUTER_SESSION_SQL,
|
||||
) -> None:
|
||||
await db.execute_raw(statement, *_upsert_params(transaction))
|
||||
|
||||
|
||||
async def _upsert_turn_with_retry(
|
||||
prisma_client: PrismaClient,
|
||||
transaction: AutoRouterTurnTransaction,
|
||||
n_retry_times: int,
|
||||
statement: str,
|
||||
) -> None:
|
||||
for attempt in range(n_retry_times + 1):
|
||||
try:
|
||||
await write_autorouter_turn(prisma_client.db, transaction)
|
||||
await write_autorouter_turn(prisma_client.db, transaction, statement)
|
||||
except DB_RETRY_SAFE_ERROR_TYPES:
|
||||
if attempt >= n_retry_times:
|
||||
raise
|
||||
|
|
@ -397,6 +433,58 @@ async def _upsert_turn_with_retry(
|
|||
return
|
||||
|
||||
|
||||
def _session_partition(transaction: AutoRouterTurnTransaction) -> tuple[str, str, str, str]:
|
||||
identity: Final = ("key", transaction.api_key) if transaction.api_key else ("user", transaction.user_id)
|
||||
return (*identity, transaction.session_id, transaction.router_name)
|
||||
|
||||
|
||||
async def _drain_session_partition(
|
||||
prisma_client: PrismaClient,
|
||||
transactions: tuple[AutoRouterTurnTransaction, ...],
|
||||
n_retry_times: int,
|
||||
statement: str,
|
||||
) -> tuple[AutoRouterTurnTransaction, ...]:
|
||||
for position, transaction in enumerate(transactions):
|
||||
try:
|
||||
await _upsert_turn_with_retry(prisma_client, transaction, n_retry_times, statement)
|
||||
except Exception as flush_err: # noqa: BLE001 # stop dependent turns without retrying an ambiguous write
|
||||
verbose_proxy_logger.error(
|
||||
"Spend tracking - auto-router session rollup flush failed for router %s; "
|
||||
"%s of %s turn writes stopped in this partition: %s",
|
||||
transaction.router_name,
|
||||
len(transactions) - position,
|
||||
len(transactions),
|
||||
flush_err,
|
||||
)
|
||||
return transactions[position:]
|
||||
return ()
|
||||
|
||||
|
||||
async def _flush_session_partition(
|
||||
prisma_client: PrismaClient,
|
||||
transactions: tuple[AutoRouterTurnTransaction, ...],
|
||||
n_retry_times: int,
|
||||
) -> None:
|
||||
failed_suffix: Final = await _drain_session_partition(
|
||||
prisma_client, transactions, n_retry_times, UPSERT_AUTOROUTER_SESSION_SQL
|
||||
)
|
||||
if not failed_suffix or not failed_suffix[0].api_key:
|
||||
return
|
||||
failed_user: Final = failed_suffix[0].user_id
|
||||
other_users: Final = sorted(
|
||||
(
|
||||
transaction
|
||||
for transaction in failed_suffix[1:]
|
||||
if transaction.user_id and transaction.user_id != failed_user
|
||||
),
|
||||
key=lambda transaction: transaction.user_id,
|
||||
)
|
||||
for _, user_turns in groupby(other_users, key=lambda transaction: transaction.user_id):
|
||||
await _drain_session_partition(
|
||||
prisma_client, tuple(user_turns), n_retry_times, UPSERT_AUTOROUTER_USER_SESSION_SQL
|
||||
)
|
||||
|
||||
|
||||
async def flush_autorouter_turn_transactions(
|
||||
prisma_client: PrismaClient,
|
||||
transactions: Sequence[AutoRouterTurnTransaction],
|
||||
|
|
@ -407,38 +495,20 @@ async def flush_autorouter_turn_transactions(
|
|||
Statements run sequentially in per-session event order: a turn's classification
|
||||
depends on the turns before it, and Postgres rejects one multi-row INSERT touching
|
||||
the same key twice. Only ConnectError is retried, per statement, because it proves
|
||||
that statement never reached the database. Any other failure drops the remaining
|
||||
turns of THAT session only, with an error log, and the flush continues with the
|
||||
next session: sessions are independent state machines, so one poisoned statement
|
||||
must not discard unrelated sessions, and a repeated increment is worse than an
|
||||
undercount. Callers must not add their own retry around this function.
|
||||
that statement never reached the database. A failed write stops its key and user
|
||||
histories for this batch. Other users sharing that key can still advance their
|
||||
independent user histories, with the key projection disabled and the real key
|
||||
identity preserved. The failed turn is never replayed. Callers must not add their
|
||||
own retry around this function.
|
||||
"""
|
||||
if not transactions:
|
||||
return
|
||||
ordered: Final = sorted(
|
||||
transactions,
|
||||
key=lambda transaction: (
|
||||
transaction.api_key,
|
||||
transaction.session_id,
|
||||
transaction.router_name,
|
||||
transaction.turn_at,
|
||||
),
|
||||
key=lambda transaction: (*_session_partition(transaction), transaction.turn_at),
|
||||
)
|
||||
for session_key, session_group in groupby(
|
||||
for _, session_group in groupby(
|
||||
ordered,
|
||||
key=lambda transaction: (transaction.api_key, transaction.session_id, transaction.router_name),
|
||||
key=_session_partition,
|
||||
):
|
||||
session_turns = tuple(session_group)
|
||||
for position, transaction in enumerate(session_turns):
|
||||
try:
|
||||
await _upsert_turn_with_retry(prisma_client, transaction, n_retry_times)
|
||||
except Exception as flush_err: # noqa: BLE001 # a statement failure drops only its session's remainder by design
|
||||
verbose_proxy_logger.error(
|
||||
"Spend tracking - auto-router session rollup flush failed for router %s; "
|
||||
"%s of %s turn transactions dropped for one session: %s",
|
||||
session_key[2],
|
||||
len(session_turns) - position,
|
||||
len(session_turns),
|
||||
flush_err,
|
||||
)
|
||||
break
|
||||
await _flush_session_partition(prisma_client, tuple(session_group), n_retry_times)
|
||||
|
|
|
|||
|
|
@ -171,6 +171,7 @@ class _Change(BaseModel):
|
|||
request_id: str
|
||||
publication: BaselinePublication
|
||||
api_key: str
|
||||
user_id: str = ""
|
||||
session_id: str
|
||||
router_name: str
|
||||
baseline_model: str
|
||||
|
|
@ -256,42 +257,54 @@ SET publication = x.publication::text
|
|||
FROM jsonb_to_recordset($1::jsonb) AS x(request_id text, publication jsonb)
|
||||
WHERE observations.request_id = x.request_id
|
||||
"""
|
||||
_UPDATE_SESSIONS: Final = """
|
||||
|
||||
|
||||
def _session_correction_sql(*, user_scoped: bool) -> str:
|
||||
table_name: Final = "LiteLLM_AutoRouterUserSession" if user_scoped else "LiteLLM_AutoRouterSession"
|
||||
identity_columns: Final = ("user_id, " if user_scoped else "") + "api_key, session_id, router_name"
|
||||
user_filter: Final = "WHERE user_id <> ''" if user_scoped else ""
|
||||
user_match: Final = "session.user_id = totals.user_id AND " if user_scoped else ""
|
||||
return f"""
|
||||
WITH changes AS (
|
||||
SELECT * FROM jsonb_to_recordset($1::jsonb) AS x(
|
||||
api_key text, session_id text, router_name text, baseline_model text,
|
||||
user_id text, api_key text, session_id text, router_name text, baseline_model text,
|
||||
covered_delta int, actual_delta float8, savings_delta float8
|
||||
)
|
||||
{user_filter}
|
||||
), totals AS (
|
||||
SELECT api_key, session_id, router_name, SUM(covered_delta)::int AS covered_delta,
|
||||
SELECT {identity_columns}, SUM(covered_delta)::int AS covered_delta,
|
||||
SUM(actual_delta) AS actual_delta, SUM(savings_delta) AS savings_delta
|
||||
FROM changes GROUP BY api_key, session_id, router_name
|
||||
FROM changes GROUP BY {identity_columns}
|
||||
), models AS (
|
||||
SELECT api_key, session_id, router_name, jsonb_object_agg(baseline_model, delta) AS deltas
|
||||
SELECT {identity_columns}, jsonb_object_agg(baseline_model, delta) AS deltas
|
||||
FROM (
|
||||
SELECT api_key, session_id, router_name, baseline_model, SUM(covered_delta)::int AS delta
|
||||
FROM changes GROUP BY api_key, session_id, router_name, baseline_model
|
||||
) grouped GROUP BY api_key, session_id, router_name
|
||||
SELECT {identity_columns}, baseline_model, SUM(covered_delta)::int AS delta
|
||||
FROM changes GROUP BY {identity_columns}, baseline_model
|
||||
) grouped GROUP BY {identity_columns}
|
||||
)
|
||||
UPDATE "LiteLLM_AutoRouterSession" AS session
|
||||
UPDATE "{table_name}" AS session
|
||||
SET saved_spend = session.saved_spend + totals.savings_delta,
|
||||
savings_estimated_turns = session.savings_estimated_turns + totals.covered_delta,
|
||||
savings_estimated_actual_spend = session.savings_estimated_actual_spend + totals.actual_delta,
|
||||
savings_estimated_saved_spend = session.savings_estimated_saved_spend + totals.savings_delta,
|
||||
savings_estimated_baseline_models = (
|
||||
SELECT COALESCE(jsonb_object_agg(key, value), '{}'::jsonb) FROM (
|
||||
SELECT COALESCE(jsonb_object_agg(key, value), '{{}}'::jsonb) FROM (
|
||||
SELECT key, SUM(value::int)::int AS value FROM (
|
||||
SELECT * FROM jsonb_each_text(session.savings_estimated_baseline_models)
|
||||
UNION ALL SELECT * FROM jsonb_each_text(models.deltas)
|
||||
) combined GROUP BY key HAVING SUM(value::int) > 0
|
||||
) counts
|
||||
)
|
||||
FROM totals JOIN models USING (api_key, session_id, router_name)
|
||||
WHERE session.api_key = totals.api_key AND session.session_id = totals.session_id
|
||||
FROM totals JOIN models USING ({identity_columns})
|
||||
WHERE {user_match}session.api_key = totals.api_key AND session.session_id = totals.session_id
|
||||
AND session.router_name = totals.router_name
|
||||
"""
|
||||
|
||||
|
||||
_UPDATE_SESSIONS: Final = _session_correction_sql(user_scoped=False)
|
||||
_UPDATE_USER_SESSIONS: Final = _session_correction_sql(user_scoped=True)
|
||||
|
||||
|
||||
def _primary_transaction(client: PrismaClient) -> _TransactionManager:
|
||||
primary: Final = cast(_TransactionalDatabase, writer_wrapper(client.db))
|
||||
return primary.tx(timeout=_TRANSACTION_TIMEOUT)
|
||||
|
|
@ -308,6 +321,7 @@ def _change(record: BaselineAccountingRecord, old: BaselinePublication | None, n
|
|||
request_id=record.observation.request_id,
|
||||
publication=new,
|
||||
api_key=record.api_key,
|
||||
user_id=record.turn.user_id if record.turn is not None else "",
|
||||
session_id=record.session_id,
|
||||
router_name=record.router_name,
|
||||
baseline_model=record.baseline_model,
|
||||
|
|
@ -357,6 +371,8 @@ async def _publish(db: SupportsRawQueries, changes: Sequence[_Change]) -> None:
|
|||
serialized: Final = json.dumps(tuple(change.model_dump(mode="json") for change in changes), separators=(",", ":"))
|
||||
await db.execute_raw(_UPDATE_LOGS, serialized)
|
||||
await db.execute_raw(_UPDATE_SESSIONS, serialized)
|
||||
if any(change.user_id for change in changes):
|
||||
await db.execute_raw(_UPDATE_USER_SESSIONS, serialized)
|
||||
for entity, table in DAILY_SPEND_TABLES.items():
|
||||
if adjustments := tuple(
|
||||
change.daily.adjustment(target, change.savings_delta, change.request_id)
|
||||
|
|
|
|||
|
|
@ -492,6 +492,18 @@ class SpendLogCleanup:
|
|||
deadline=deadline,
|
||||
)
|
||||
|
||||
async def _delete_old_autorouter_user_session_rows(
|
||||
self, prisma_client: PrismaClient, cutoff_date: datetime, deadline: float
|
||||
) -> TableCleanupResult:
|
||||
return await self._delete_old_rows_batched(
|
||||
prisma_client,
|
||||
cutoff_date,
|
||||
table_name="LiteLLM_AutoRouterUserSession",
|
||||
key_columns=("user_id", "api_key", "session_id", "router_name"),
|
||||
time_column="last_turn_at",
|
||||
deadline=deadline,
|
||||
)
|
||||
|
||||
async def _delete_old_health_check_rows(
|
||||
self, prisma_client: PrismaClient, cutoff_date: datetime, deadline: float
|
||||
) -> TableCleanupResult:
|
||||
|
|
@ -560,9 +572,17 @@ class SpendLogCleanup:
|
|||
)
|
||||
except Exception: # noqa: BLE001 # retained observations are retried by the next cleanup job
|
||||
verbose_proxy_logger.warning("Auto-router baseline retention remains pending")
|
||||
sessions_result: Final = await self._delete_old_autorouter_session_rows(prisma_client, session_cutoff, deadline)
|
||||
sessions_result: Final = await self._delete_old_autorouter_session_rows(
|
||||
prisma_client, session_cutoff, self._group_deadline(deadline, 2)
|
||||
)
|
||||
verbose_proxy_logger.info("Deleted %s expired auto-router session rollup rows", sessions_result.rows_deleted)
|
||||
return (sessions_result,)
|
||||
user_sessions_result: Final = await self._delete_old_autorouter_user_session_rows(
|
||||
prisma_client, session_cutoff, deadline
|
||||
)
|
||||
verbose_proxy_logger.info(
|
||||
"Deleted %s expired auto-router user session rollup rows", user_sessions_result.rows_deleted
|
||||
)
|
||||
return (sessions_result, user_sessions_result)
|
||||
|
||||
async def _clean_health_checks(
|
||||
self, prisma_client: PrismaClient, retention_seconds: int, deadline: float
|
||||
|
|
|
|||
|
|
@ -746,14 +746,18 @@ async def get_auto_router_benchmarks(
|
|||
] = None,
|
||||
end_date: Annotated[str | None, Query(description="YYYY-MM-DD UTC, inclusive (defaults to today)")] = None,
|
||||
api_key: Annotated[str | None, Query(description="Filter to one virtual key token hash")] = None,
|
||||
user_id: Annotated[
|
||||
str | None, Query(min_length=1, description="Filter to one canonical internal user recorded on each turn")
|
||||
] = None,
|
||||
) -> AutoRouterBenchmarksResponse:
|
||||
"""
|
||||
Benchmarks for the auto-router dashboard: session shape, savings against the configured
|
||||
baseline, and prompt-caching behaviour bucketed by what the router did.
|
||||
|
||||
Reads the LiteLLM_AutoRouterSession rollup, folded once per request at spend-write time,
|
||||
so this endpoint never scans LiteLLM_SpendLogs. A session is in the window when it
|
||||
overlaps it: its last turn is on or after start_date and its first turn is on or before
|
||||
Reads session rollups folded once per request at spend-write time, so this endpoint
|
||||
never scans LiteLLM_SpendLogs. A user filter selects only turns attributed to that
|
||||
internal user when written; older key-only history remains outside user views. A session
|
||||
is in the window when it overlaps it: its last turn is on or after start_date and its first turn is on or before
|
||||
end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is
|
||||
over that bucket's turns.
|
||||
|
||||
|
|
@ -783,6 +787,7 @@ async def get_auto_router_benchmarks(
|
|||
start_day.isoformat(),
|
||||
(end_day + timedelta(days=1)).isoformat(),
|
||||
api_key,
|
||||
user_id,
|
||||
)
|
||||
rows: Final = _SESSION_AGG_ROWS.validate_python(raw_rows or ())
|
||||
groups: Final = (
|
||||
|
|
|
|||
|
|
@ -1620,6 +1620,47 @@ model LiteLLM_AutoRouterSession {
|
|||
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterUserSession {
|
||||
user_id String
|
||||
api_key String
|
||||
session_id String
|
||||
router_name String
|
||||
router_type String
|
||||
first_turn_at DateTime
|
||||
last_turn_at DateTime
|
||||
last_model String
|
||||
models Json @default("{}")
|
||||
turns Int @default(0)
|
||||
unordered_turns Int @default(0)
|
||||
covered_turns Int @default(0)
|
||||
cache_hits Int @default(0)
|
||||
same_model_turns Int @default(0)
|
||||
same_model_hits Int @default(0)
|
||||
first_visit_turns Int @default(0)
|
||||
first_visit_hits Int @default(0)
|
||||
return_turns Int @default(0)
|
||||
return_hits Int @default(0)
|
||||
return_expired_misses Int @default(0)
|
||||
return_within_ttl_misses Int @default(0)
|
||||
ttl_5m_turns Int @default(0)
|
||||
ttl_1h_turns Int @default(0)
|
||||
total_tokens BigInt @default(0)
|
||||
spend Float @default(0)
|
||||
saved_spend Float @default(0)
|
||||
savings_estimated_turns Int @default(0)
|
||||
savings_estimated_actual_spend Float @default(0)
|
||||
savings_estimated_saved_spend Float @default(0)
|
||||
savings_estimated_baseline_models Json @default("{}")
|
||||
classifier_cost Float @default(0)
|
||||
classifier_cost_recorded_turns Int @default(0)
|
||||
tier_turns Json @default("{}")
|
||||
baseline_models Json @default("{}")
|
||||
|
||||
@@id([user_id, api_key, session_id, router_name])
|
||||
@@index([last_turn_at], map: "idx_autorouter_user_session_last_turn")
|
||||
@@index([user_id, last_turn_at], map: "idx_autorouter_user_session_user_last_turn")
|
||||
}
|
||||
|
||||
// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
|
||||
// either direction. forward duplicates the requests the keys did not route through the
|
||||
// router through it, answering whether they should adopt it; reverse duplicates the
|
||||
|
|
|
|||
|
|
@ -1620,6 +1620,47 @@ model LiteLLM_AutoRouterSession {
|
|||
@@index([last_turn_at], map: "idx_autorouter_session_last_turn")
|
||||
}
|
||||
|
||||
model LiteLLM_AutoRouterUserSession {
|
||||
user_id String
|
||||
api_key String
|
||||
session_id String
|
||||
router_name String
|
||||
router_type String
|
||||
first_turn_at DateTime
|
||||
last_turn_at DateTime
|
||||
last_model String
|
||||
models Json @default("{}")
|
||||
turns Int @default(0)
|
||||
unordered_turns Int @default(0)
|
||||
covered_turns Int @default(0)
|
||||
cache_hits Int @default(0)
|
||||
same_model_turns Int @default(0)
|
||||
same_model_hits Int @default(0)
|
||||
first_visit_turns Int @default(0)
|
||||
first_visit_hits Int @default(0)
|
||||
return_turns Int @default(0)
|
||||
return_hits Int @default(0)
|
||||
return_expired_misses Int @default(0)
|
||||
return_within_ttl_misses Int @default(0)
|
||||
ttl_5m_turns Int @default(0)
|
||||
ttl_1h_turns Int @default(0)
|
||||
total_tokens BigInt @default(0)
|
||||
spend Float @default(0)
|
||||
saved_spend Float @default(0)
|
||||
savings_estimated_turns Int @default(0)
|
||||
savings_estimated_actual_spend Float @default(0)
|
||||
savings_estimated_saved_spend Float @default(0)
|
||||
savings_estimated_baseline_models Json @default("{}")
|
||||
classifier_cost Float @default(0)
|
||||
classifier_cost_recorded_turns Int @default(0)
|
||||
tier_turns Json @default("{}")
|
||||
baseline_models Json @default("{}")
|
||||
|
||||
@@id([user_id, api_key, session_id, router_name])
|
||||
@@index([last_turn_at], map: "idx_autorouter_user_session_last_turn")
|
||||
@@index([user_id, last_turn_at], map: "idx_autorouter_user_session_user_last_turn")
|
||||
}
|
||||
|
||||
// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
|
||||
// either direction. forward duplicates the requests the keys did not route through the
|
||||
// router through it, answering whether they should adopt it; reverse duplicates the
|
||||
|
|
|
|||
|
|
@ -6,17 +6,24 @@ tests/test_litellm/proxy/db/test_autorouter_session_rollup.py.
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Final
|
||||
from types import SimpleNamespace
|
||||
from typing import Final, TypedDict, cast
|
||||
|
||||
import pytest
|
||||
from prisma import Prisma
|
||||
from prisma.errors import RawQueryError
|
||||
from typing_extensions import ReadOnly
|
||||
|
||||
from litellm.proxy.db.autorouter_session_rollup import (
|
||||
AUTOROUTER_BENCHMARKS_SQL,
|
||||
UPSERT_AUTOROUTER_SESSION_SQL,
|
||||
AutoRouterTurnTransaction,
|
||||
flush_autorouter_turn_transactions,
|
||||
)
|
||||
from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import SpendLogCleanup
|
||||
|
||||
pytestmark = pytest.mark.asyncio(loop_scope="session")
|
||||
|
||||
|
|
@ -45,6 +52,7 @@ async def _turn(
|
|||
tier: "str | None" = None,
|
||||
baseline: "str | None" = None,
|
||||
estimated: bool = True,
|
||||
user_id: str = "",
|
||||
) -> None:
|
||||
touched: Final = 1 if (hit or ttl is not None or not covered) else 0
|
||||
await db.execute_raw(
|
||||
|
|
@ -68,6 +76,7 @@ async def _turn(
|
|||
int(estimated),
|
||||
spend if estimated else 0.0,
|
||||
saved if estimated else 0.0,
|
||||
user_id,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -217,7 +226,7 @@ async def test_subtotal_coverage_survives_legacy_and_rolling_writers(db, writers
|
|||
assert row["savings_estimated_actual_spend"] == pytest.approx(0.01 * sum(writers))
|
||||
assert row["savings_estimated_saved_spend"] == pytest.approx(0.02 * sum(writers))
|
||||
groups: Final = await db.query_raw(
|
||||
AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key
|
||||
AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key, None
|
||||
)
|
||||
assert len(groups) == 1
|
||||
assert groups[0]["classifier_cost"] == row["classifier_cost"]
|
||||
|
|
@ -242,7 +251,7 @@ async def test_unknown_and_legacy_turns_preserve_actual_spend_without_entering_t
|
|||
assert row["saved_spend"] == pytest.approx(-0.03)
|
||||
assert row["savings_estimated_baseline_models"] == {"opus": 1}
|
||||
groups: Final = await db.query_raw(
|
||||
AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key
|
||||
AUTOROUTER_BENCHMARKS_SQL, T0.isoformat(), (T0 + timedelta(days=1)).isoformat(), key, None
|
||||
)
|
||||
assert len(groups) == 1
|
||||
for actual in (row, groups[0]):
|
||||
|
|
@ -277,6 +286,7 @@ async def test_the_benchmarks_aggregate_reads_only_overlapping_sessions(db):
|
|||
(T0 - timedelta(days=1)).isoformat(),
|
||||
(T0 + timedelta(days=1)).isoformat(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
matching = [row for row in rows if row["router_name"] == router]
|
||||
assert len(matching) == 1
|
||||
|
|
@ -304,6 +314,7 @@ async def test_the_benchmarks_aggregate_can_filter_to_one_key(db):
|
|||
(T0 - timedelta(days=1)).isoformat(),
|
||||
(T0 + timedelta(days=1)).isoformat(),
|
||||
first_key,
|
||||
None,
|
||||
)
|
||||
matching = [row for row in rows if row["router_name"] == router]
|
||||
assert len(matching) == 1
|
||||
|
|
@ -317,10 +328,160 @@ async def test_the_benchmarks_aggregate_can_filter_to_one_key(db):
|
|||
(T0 - timedelta(days=1)).isoformat(),
|
||||
(T0 + timedelta(days=1)).isoformat(),
|
||||
f"k-{uuid.uuid4()}",
|
||||
None,
|
||||
)
|
||||
assert [row for row in unknown_key_rows if row["router_name"] == router] == []
|
||||
|
||||
|
||||
class _BenchmarkRow(TypedDict):
|
||||
sessions: ReadOnly[int]
|
||||
turns: ReadOnly[int]
|
||||
same_model_turns: ReadOnly[int]
|
||||
first_visit_turns: ReadOnly[int]
|
||||
spend: ReadOnly[float]
|
||||
saved_spend: ReadOnly[float]
|
||||
tier_turns: ReadOnly[dict[str, int]]
|
||||
cache_hits: ReadOnly[int]
|
||||
savings_estimated_turns: ReadOnly[int]
|
||||
savings_estimated_actual_spend: ReadOnly[float]
|
||||
savings_estimated_saved_spend: ReadOnly[float]
|
||||
|
||||
|
||||
async def _scoped_benchmarks(
|
||||
db: Prisma, router: str, user_id: str | None = None, key: str | None = None
|
||||
) -> tuple[_BenchmarkRow, ...]:
|
||||
rows: Final = await db.query_raw(
|
||||
AUTOROUTER_BENCHMARKS_SQL,
|
||||
(T0 - timedelta(days=1)).isoformat(),
|
||||
(T0 + timedelta(days=1)).isoformat(),
|
||||
key,
|
||||
user_id,
|
||||
)
|
||||
return tuple(cast(_BenchmarkRow, row) for row in rows if row["router_name"] == router)
|
||||
|
||||
|
||||
async def test_users_keep_written_identity_across_shared_keys_and_keyless_sessions(db: Prisma) -> None:
|
||||
router: Final = f"r-{uuid.uuid4()}"
|
||||
alice: Final = f"u-{uuid.uuid4()}"
|
||||
bob: Final = f"u-{uuid.uuid4()}"
|
||||
first_key: Final = f"k-{uuid.uuid4()}"
|
||||
second_key: Final = f"k-{uuid.uuid4()}"
|
||||
await _legacy_turn(db, first_key, T0, router=router)
|
||||
await _turn(db, first_key, "A", T0 + timedelta(seconds=10), router=router, user_id=alice, tier="simple")
|
||||
await _turn(
|
||||
db, first_key, "B", T0 + timedelta(seconds=20), router=router, user_id=bob, spend=0.03, saved=0.06, tier="complex"
|
||||
)
|
||||
await _turn(db, second_key, "C", T0, router=router, user_id=alice, spend=0.02, saved=0.04)
|
||||
await _turn(db, "", "A", T0, router=router, user_id=alice, ttl=300)
|
||||
await _turn(db, "", "A", T0 + timedelta(seconds=1), router=router, user_id=alice, hit=1)
|
||||
await _turn(db, "", "B", T0, router=router, user_id=bob, spend=0.04, saved=0.08)
|
||||
await _turn(db, second_key, "C", T0 - timedelta(days=40), router=router, user_id=alice, session_id="expired")
|
||||
|
||||
alice_rows: Final = await _scoped_benchmarks(db, router, user_id=alice)
|
||||
bob_rows: Final = await _scoped_benchmarks(db, router, user_id=bob)
|
||||
global_rows: Final = await _scoped_benchmarks(db, router)
|
||||
key_rows: Final = await _scoped_benchmarks(db, router, key=first_key)
|
||||
intersection: Final = await _scoped_benchmarks(db, router, user_id=alice, key=first_key)
|
||||
assert len(alice_rows) == len(bob_rows) == len(global_rows) == len(key_rows) == len(intersection) == 1
|
||||
assert (alice_rows[0]["sessions"], alice_rows[0]["turns"], alice_rows[0]["same_model_turns"]) == (3, 4, 1)
|
||||
assert (bob_rows[0]["sessions"], bob_rows[0]["turns"], bob_rows[0]["first_visit_turns"]) == (2, 2, 2)
|
||||
assert alice_rows[0]["spend"] == pytest.approx(0.05)
|
||||
assert bob_rows[0]["spend"] == pytest.approx(0.07)
|
||||
assert alice_rows[0]["tier_turns"] == {"simple": 1}
|
||||
assert bob_rows[0]["tier_turns"] == {"complex": 1}
|
||||
assert (alice_rows[0]["cache_hits"], bob_rows[0]["cache_hits"]) == (1, 0)
|
||||
assert (global_rows[0]["sessions"], global_rows[0]["turns"]) == (4, 7)
|
||||
assert (alice_rows[0]["savings_estimated_turns"], bob_rows[0]["savings_estimated_turns"]) == (4, 2)
|
||||
assert global_rows[0]["savings_estimated_turns"] == 6
|
||||
for scoped in (alice_rows[0], bob_rows[0]):
|
||||
assert scoped["savings_estimated_actual_spend"] == pytest.approx(scoped["spend"])
|
||||
assert scoped["savings_estimated_saved_spend"] == pytest.approx(scoped["saved_spend"])
|
||||
assert global_rows[0]["spend"] == pytest.approx(alice_rows[0]["spend"] + bob_rows[0]["spend"] + 0.01)
|
||||
assert global_rows[0]["saved_spend"] == pytest.approx(alice_rows[0]["saved_spend"] + bob_rows[0]["saved_spend"] + 0.02)
|
||||
assert global_rows[0]["tier_turns"] == {"simple": 1, "complex": 1}
|
||||
assert (key_rows[0]["sessions"], key_rows[0]["turns"]) == (1, 3)
|
||||
assert key_rows[0]["spend"] == pytest.approx(0.05)
|
||||
assert (intersection[0]["sessions"], intersection[0]["turns"]) == (1, 1)
|
||||
assert intersection[0]["spend"] == pytest.approx(0.01)
|
||||
assert await _scoped_benchmarks(db, router, user_id=bob, key=second_key) == ()
|
||||
assert await _scoped_benchmarks(db, router, user_id=f"u-{uuid.uuid4()}") == ()
|
||||
assert await _scoped_benchmarks(db, router, user_id="") == ()
|
||||
|
||||
|
||||
async def test_a_failed_user_projection_rolls_back_the_keys_increment(db: Prisma) -> None:
|
||||
key: Final = f"k-{uuid.uuid4()}"
|
||||
user_id: Final = "".join(str(uuid.uuid4()) for _ in range(200))
|
||||
await _turn(db, key, "A", T0)
|
||||
before: Final = await _row(db, key)
|
||||
|
||||
with pytest.raises(RawQueryError, match=r"index row (requires|size)"):
|
||||
await _turn(db, key, "B", T0 + timedelta(seconds=1), user_id=user_id)
|
||||
|
||||
assert await _row(db, key) == before
|
||||
assert await db.query_raw('SELECT user_id FROM "LiteLLM_AutoRouterUserSession" WHERE user_id = $1', user_id) == []
|
||||
|
||||
first_user: Final = f"u-{uuid.uuid4()}"
|
||||
second_user: Final = f"u-{uuid.uuid4()}"
|
||||
turns: Final = tuple(
|
||||
AutoRouterTurnTransaction(
|
||||
api_key=key,
|
||||
user_id=user,
|
||||
session_id="s1",
|
||||
router_name="auto-1",
|
||||
router_type="complexity",
|
||||
model=model,
|
||||
turn_at=T0 + timedelta(seconds=second),
|
||||
total_tokens=100,
|
||||
spend=0.01,
|
||||
saved_spend=0.02,
|
||||
classifier_cost=0.0,
|
||||
covered=True,
|
||||
cache_hit=False,
|
||||
cache_ttl_seconds=None,
|
||||
cache_touched=False,
|
||||
)
|
||||
for user, model, second in (
|
||||
(first_user, "A", 1),
|
||||
(user_id, "B", 2),
|
||||
(first_user, "B", 3),
|
||||
(second_user, "C", 4),
|
||||
(first_user, "B", 5),
|
||||
(second_user, "C", 6),
|
||||
(user_id, "A", 7),
|
||||
)
|
||||
)
|
||||
await flush_autorouter_turn_transactions(SimpleNamespace(db=db), tuple(reversed(turns)), n_retry_times=0)
|
||||
|
||||
key_row: Final = await _row(db, key)
|
||||
assert (key_row["turns"], key_row["last_model"], key_row["unordered_turns"]) == (2, "A", 0)
|
||||
assert key_row["spend"] == pytest.approx(0.02)
|
||||
user_rows: Final = await db.query_raw('SELECT * FROM "LiteLLM_AutoRouterUserSession" WHERE api_key = $1', key)
|
||||
by_user: Final = {row["user_id"]: row for row in user_rows}
|
||||
assert set(by_user) == {first_user, second_user}
|
||||
for user, count, model in ((first_user, 3, "B"), (second_user, 2, "C")):
|
||||
row: Final = by_user[user]
|
||||
assert (row["turns"], row["same_model_turns"], row["unordered_turns"], row["last_model"]) == (count, 1, 0, model)
|
||||
assert row["spend"] == pytest.approx(count * 0.01)
|
||||
assert row["saved_spend"] == pytest.approx(count * 0.02)
|
||||
|
||||
|
||||
async def test_user_session_cleanup_keeps_another_users_recent_keyless_session(db: Prisma) -> None:
|
||||
router: Final = f"r-{uuid.uuid4()}"
|
||||
expired_user: Final = f"u-{uuid.uuid4()}"
|
||||
recent_user: Final = f"u-{uuid.uuid4()}"
|
||||
await _turn(db, "", "A", T0 - timedelta(days=1), router=router, user_id=expired_user)
|
||||
await _turn(db, "", "A", T0 + timedelta(days=1), router=router, user_id=recent_user)
|
||||
cleaner: Final = SpendLogCleanup(general_settings={})
|
||||
|
||||
await cleaner._delete_old_autorouter_user_session_rows(
|
||||
SimpleNamespace(db=db), T0.replace(tzinfo=timezone.utc), time.monotonic() + 60
|
||||
)
|
||||
|
||||
assert await db.query_raw(
|
||||
'SELECT user_id, turns FROM "LiteLLM_AutoRouterUserSession" WHERE router_name = $1', router
|
||||
) == [{"user_id": recent_user, "turns": 1}]
|
||||
|
||||
|
||||
async def test_a_reconfigured_alias_reports_each_router_type_as_its_own_group(db):
|
||||
key = f"k-{uuid.uuid4()}"
|
||||
router = f"r-{uuid.uuid4()}"
|
||||
|
|
@ -334,6 +495,7 @@ async def test_a_reconfigured_alias_reports_each_router_type_as_its_own_group(db
|
|||
(T0 - timedelta(days=1)).isoformat(),
|
||||
(T0 + timedelta(days=1)).isoformat(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
matching = sorted(
|
||||
(row for row in rows if row["router_name"] == router),
|
||||
|
|
@ -418,6 +580,7 @@ async def test_the_benchmarks_aggregate_sums_tier_turns_across_sessions(db):
|
|||
(T0 - timedelta(days=1)).isoformat(),
|
||||
(T0 + timedelta(days=1)).isoformat(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
grouped = next(row for row in rows if row["router_name"] == router)
|
||||
assert grouped["tier_turns"] == {"simple": 2, "complex": 1}
|
||||
|
|
@ -446,6 +609,7 @@ async def test_tier_maps_stay_separate_per_router_type_on_a_reconfigured_alias(d
|
|||
(T0 - timedelta(days=1)).isoformat(),
|
||||
(T0 + timedelta(days=1)).isoformat(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
by_type = {row["router_type"]: row["tier_turns"] for row in rows if row["router_name"] == router}
|
||||
assert by_type == {"complexity": {"medium": 1}, "quality": {"2": 1}}
|
||||
|
|
@ -461,6 +625,7 @@ async def test_a_window_with_no_tiered_turns_aggregates_to_an_empty_map(db):
|
|||
(T0 - timedelta(days=1)).isoformat(),
|
||||
(T0 + timedelta(days=1)).isoformat(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
grouped = next(row for row in rows if row["router_name"] == router)
|
||||
assert grouped["tier_turns"] == {}
|
||||
|
|
|
|||
|
|
@ -56,7 +56,9 @@ def record() -> Callable[..., BaselineAccountingRecord]:
|
|||
},
|
||||
)
|
||||
|
||||
def create(label: str = "first", started: float = 10000.0, identical: bool = True) -> BaselineAccountingRecord:
|
||||
def create(
|
||||
label: str = "first", started: float = 10000.0, identical: bool = True, user_id: str = ""
|
||||
) -> BaselineAccountingRecord:
|
||||
return BaselineAccountingRecord(
|
||||
scope="autorouter-baseline:v3:" + run * 2, api_key=run, session_id=run,
|
||||
router_name="test-router", baseline_model="anthropic/claude-opus-5",
|
||||
|
|
@ -76,6 +78,7 @@ def record() -> Callable[..., BaselineAccountingRecord]:
|
|||
total_tokens=6230, spend=0.17, saved_spend=0.0, classifier_cost=0.0,
|
||||
covered=True, cache_hit=False, cache_ttl_seconds=3600, cache_touched=True,
|
||||
baseline_model="anthropic/claude-opus-5",
|
||||
user_id=user_id,
|
||||
),
|
||||
daily=DailyBaselineAttribution(
|
||||
date="2026-09-15", api_key=run, model="claude-opus-5", custom_llm_provider="anthropic",
|
||||
|
|
@ -99,21 +102,36 @@ async def _session(db: Prisma, record: BaselineAccountingRecord):
|
|||
return rows[0]
|
||||
|
||||
|
||||
async def _user_sessions(db: Prisma, record: BaselineAccountingRecord) -> dict[str, dict[str, object]]:
|
||||
rows: Final = await db.query_raw('SELECT * FROM "LiteLLM_AutoRouterUserSession" WHERE api_key=$1', record.api_key)
|
||||
return {str(row["user_id"]): row for row in rows}
|
||||
|
||||
|
||||
async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
|
||||
store: Final = _store(db)
|
||||
late: Final = record("late", 10001.0)
|
||||
early: Final = record("early", identical=False)
|
||||
late: Final = record("late", 10001.0, user_id="late-user")
|
||||
early: Final = record("early", identical=False, user_id="early-user")
|
||||
await _log(db, late)
|
||||
assert await store.append(late) == "recorded"
|
||||
assert await store.project(late.scope) == "published"
|
||||
before: Final = await _session(db, late)
|
||||
assert before["savings_estimated_actual_spend"] == before["spend"] == 0.17
|
||||
assert before["saved_spend"] == 0.0
|
||||
before_users: Final = await _user_sessions(db, late)
|
||||
assert set(before_users) == {"late-user"}
|
||||
assert before_users["late-user"]["savings_estimated_turns"] == 1
|
||||
assert before_users["late-user"]["savings_estimated_baseline_models"] == {late.baseline_model: 1}
|
||||
await _log(db, early)
|
||||
assert await store.append(early) == "recorded"
|
||||
pending: Final = await _session(db, late)
|
||||
assert pending["spend"] == 0.34 and pending["savings_estimated_turns"] == 0
|
||||
assert pending["saved_spend"] == pending["savings_estimated_actual_spend"] == 0.0
|
||||
pending_users: Final = await _user_sessions(db, late)
|
||||
assert set(pending_users) == {"late-user", "early-user"}
|
||||
for user in pending_users.values():
|
||||
assert user["turns"] == 1 and user["spend"] == 0.17
|
||||
assert user["savings_estimated_turns"] == user["savings_estimated_actual_spend"] == user["saved_spend"] == 0
|
||||
assert user["savings_estimated_baseline_models"] == {}
|
||||
waiting: Final = await db.query_raw('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=$1', late.observation.request_id)
|
||||
assert waiting[0]["metadata"]["autorouter_savings"] is None
|
||||
assert waiting[0]["metadata"]["autorouter_savings_estimate"]["reason"] == "pending_projection"
|
||||
|
|
@ -125,37 +143,69 @@ async def test_late_replay_updates_all_projections_without_rebilling(db: Prisma,
|
|||
assert logs[0]["spend"] == 0.17
|
||||
assert logs[0]["metadata"]["autorouter_savings_estimate"]["provenance"] == "modeled"
|
||||
assert after["saved_spend"] == pytest.approx(logs[0]["metadata"]["autorouter_savings"])
|
||||
after_users: Final = await _user_sessions(db, late)
|
||||
assert after_users["early-user"] == pending_users["early-user"]
|
||||
for field in (
|
||||
"saved_spend", "savings_estimated_turns", "savings_estimated_actual_spend",
|
||||
"savings_estimated_saved_spend", "savings_estimated_baseline_models",
|
||||
):
|
||||
assert after_users["late-user"][field] == after[field]
|
||||
assert after_users["late-user"]["turns"] == 1 and after_users["late-user"]["spend"] == 0.17
|
||||
for table in ("DailyUserSpend", "DailyTeamSpend", "DailyOrganizationSpend", "DailyEndUserSpend", "DailyAgentSpend", "DailyTagSpend"):
|
||||
rows: Final = await db.query_raw(f'SELECT spend,api_requests,autorouter_savings_spend FROM "LiteLLM_{table}" WHERE api_key=$1', late.api_key)
|
||||
assert rows[0]["spend"] == rows[0]["api_requests"] == 0
|
||||
assert rows[0]["autorouter_savings_spend"] == pytest.approx(after["saved_spend"])
|
||||
|
||||
|
||||
async def test_commit_ack_loss_and_concurrent_duplicate_delivery_are_idempotent(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
|
||||
event: Final = record()
|
||||
@pytest.mark.parametrize("attributed", [True, False])
|
||||
async def test_commit_ack_loss_and_concurrent_duplicate_delivery_are_idempotent(
|
||||
db: Prisma, record: Callable[..., BaselineAccountingRecord], attributed: bool
|
||||
) -> None:
|
||||
event: Final = record(user_id="first-user" if attributed else "")
|
||||
other: Final = record("other", 10001.0, user_id="second-user" if attributed else "")
|
||||
await _log(db, event)
|
||||
assert await _store(db, after_commit=True).append(event) == "unavailable"
|
||||
store: Final = _store(db)
|
||||
assert set(await asyncio.gather(*(store.append(event) for _ in range(4)))) == {"recorded"}
|
||||
await _log(db, other)
|
||||
assert await store.append(other) == "recorded"
|
||||
if not attributed:
|
||||
await db.execute_raw(
|
||||
'UPDATE "LiteLLM_AutoRouterBaselineObservation" SET data=(data::jsonb #- \'{turn,user_id}\')::text WHERE scope=$1',
|
||||
event.scope,
|
||||
)
|
||||
assert await store.project(event.scope) == "published"
|
||||
assert await store.project(event.scope) == "unchanged"
|
||||
session: Final = await _session(db, event)
|
||||
assert session["turns"] == session["savings_estimated_turns"] == 1
|
||||
assert session["spend"] == session["savings_estimated_actual_spend"] == 0.17
|
||||
assert session["turns"] == session["savings_estimated_turns"] == 2
|
||||
assert session["spend"] == session["savings_estimated_actual_spend"] == 0.34
|
||||
users: Final = await _user_sessions(db, event)
|
||||
assert set(users) == ({"first-user", "second-user"} if attributed else set())
|
||||
for user in users.values():
|
||||
assert user["turns"] == user["savings_estimated_turns"] == 1
|
||||
assert user["spend"] == user["savings_estimated_actual_spend"] == 0.17
|
||||
assert user["savings_estimated_baseline_models"] == {event.baseline_model: 1}
|
||||
|
||||
|
||||
async def test_publication_rollback_keeps_dirty_revision_for_retry(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
|
||||
event: Final = record()
|
||||
event: Final = record(user_id="rollback-user")
|
||||
await _log(db, event)
|
||||
store: Final = _store(db)
|
||||
assert await store.append(event) == "recorded"
|
||||
assert await _store(db, before_commit=True).project(event.scope) == "unavailable"
|
||||
session: Final = await _session(db, event)
|
||||
assert session["spend"] == 0.17 and session["savings_estimated_turns"] == 0
|
||||
before_users: Final = await _user_sessions(db, event)
|
||||
assert before_users["rollback-user"]["spend"] == 0.17
|
||||
assert before_users["rollback-user"]["savings_estimated_turns"] == 0
|
||||
assert before_users["rollback-user"]["savings_estimated_baseline_models"] == {}
|
||||
revisions: Final = await db.query_raw('SELECT revision,published_revision FROM "LiteLLM_AutoRouterBaselineComparison" WHERE scope=$1', event.scope)
|
||||
assert revisions[0]["revision"] > revisions[0]["published_revision"]
|
||||
assert await store.project(event.scope) == "published"
|
||||
assert (await _session(db, event))["savings_estimated_turns"] == 1
|
||||
after_users: Final = await _user_sessions(db, event)
|
||||
assert after_users["rollback-user"]["turns"] == after_users["rollback-user"]["savings_estimated_turns"] == 1
|
||||
assert after_users["rollback-user"]["spend"] == after_users["rollback-user"]["savings_estimated_actual_spend"] == 0.17
|
||||
|
||||
|
||||
async def test_conflicting_duplicate_cannot_restore_an_observed_estimate(db: Prisma, record: Callable[..., BaselineAccountingRecord]) -> None:
|
||||
|
|
@ -196,6 +246,7 @@ async def test_native_observation_enters_spend_pipeline_once_with_shared_daily_a
|
|||
db: Prisma, record: Callable[..., BaselineAccountingRecord], monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import os
|
||||
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
|
||||
from litellm.proxy.hooks.autorouter_baseline_cache import CapturedBaselineObservation
|
||||
|
|
|
|||
|
|
@ -56,6 +56,31 @@ def _build(payload: dict | None = None, metadata: dict | None = None):
|
|||
|
||||
|
||||
class TestBuildTransaction:
|
||||
@pytest.mark.parametrize(
|
||||
"api_key, user_id, included",
|
||||
[
|
||||
("hashed-key", "canonical-user", True),
|
||||
("hashed-key", None, True),
|
||||
("hashed-key", "", True),
|
||||
("", "canonical-user", True),
|
||||
("", None, False),
|
||||
("", "", False),
|
||||
],
|
||||
)
|
||||
def test_attribution_uses_the_canonical_user_even_without_a_key(
|
||||
self, api_key: str, user_id: str | None, included: bool
|
||||
) -> None:
|
||||
transaction: Final = _build(
|
||||
payload=_payload(api_key=api_key, user=user_id),
|
||||
metadata=_metadata(user="client-user", user_api_key_user_id="metadata-user"),
|
||||
)
|
||||
if not included:
|
||||
assert transaction is None
|
||||
return
|
||||
assert transaction is not None
|
||||
assert transaction.api_key == api_key
|
||||
assert transaction.user_id == (user_id or "")
|
||||
|
||||
def test_successful_auto_routed_turn_builds_every_field(self):
|
||||
transaction = _build(
|
||||
metadata=_metadata(
|
||||
|
|
@ -205,23 +230,43 @@ class TestBuildTransaction:
|
|||
|
||||
|
||||
class _FakeDB:
|
||||
def __init__(self, failures: "list[Exception] | None" = None, poison_session: str | None = None):
|
||||
def __init__(
|
||||
self,
|
||||
failures: "list[Exception] | None" = None,
|
||||
poison_session: str | None = None,
|
||||
poison_user: str | None = None,
|
||||
commit_then_error_users: frozenset[str] = frozenset(),
|
||||
):
|
||||
self.calls: list[tuple] = []
|
||||
self.attempts: list[tuple[str, tuple[object, ...]]] = []
|
||||
self._failures = list(failures or [])
|
||||
self._poison_session = poison_session
|
||||
self._poison_user = poison_user
|
||||
self._commit_then_error_users = commit_then_error_users
|
||||
|
||||
async def execute_raw(self, sql: str, *params: object) -> int:
|
||||
self.attempts.append((sql, params))
|
||||
if self._poison_session is not None and params[1] == self._poison_session:
|
||||
raise RuntimeError("index row size exceeds btree maximum")
|
||||
if self._poison_user is not None and params[19] == self._poison_user:
|
||||
raise RuntimeError("index row size exceeds btree maximum")
|
||||
if self._failures:
|
||||
raise self._failures.pop(0)
|
||||
self.calls.append((sql, params))
|
||||
if params[19] in self._commit_then_error_users:
|
||||
raise RuntimeError("commit succeeded but acknowledgement was lost")
|
||||
return 1
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, failures: "list[Exception] | None" = None, poison_session: str | None = None):
|
||||
self.db = _FakeDB(failures, poison_session)
|
||||
def __init__(
|
||||
self,
|
||||
failures: "list[Exception] | None" = None,
|
||||
poison_session: str | None = None,
|
||||
poison_user: str | None = None,
|
||||
commit_then_error_users: frozenset[str] = frozenset(),
|
||||
):
|
||||
self.db = _FakeDB(failures, poison_session, poison_user, commit_then_error_users)
|
||||
|
||||
|
||||
def _transaction(
|
||||
|
|
@ -229,9 +274,11 @@ def _transaction(
|
|||
at: datetime = datetime(2026, 8, 1, 12, 0, 0),
|
||||
tier: str | None = "medium",
|
||||
baseline_model: str | None = "anthropic/claude-opus-5",
|
||||
api_key: str = "k1",
|
||||
user_id: str = "",
|
||||
) -> AutoRouterTurnTransaction:
|
||||
return AutoRouterTurnTransaction(
|
||||
api_key="k1",
|
||||
api_key=api_key,
|
||||
session_id=session_id,
|
||||
router_name="live-auto",
|
||||
router_type="complexity",
|
||||
|
|
@ -247,6 +294,7 @@ def _transaction(
|
|||
cache_touched=False,
|
||||
tier=tier,
|
||||
baseline_model=baseline_model,
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -261,7 +309,7 @@ class TestFlush:
|
|||
|
||||
def test_params_marshal_in_statement_order(self):
|
||||
client = _FakeClient()
|
||||
asyncio.run(flush_autorouter_turn_transactions(client, [_transaction()]))
|
||||
asyncio.run(flush_autorouter_turn_transactions(client, [_transaction(user_id="canonical-user")]))
|
||||
sql, params = client.db.calls[0]
|
||||
assert sql == UPSERT_AUTOROUTER_SESSION_SQL
|
||||
assert params == (
|
||||
|
|
@ -284,8 +332,65 @@ class TestFlush:
|
|||
0,
|
||||
0.0,
|
||||
0.0,
|
||||
"canonical-user",
|
||||
)
|
||||
|
||||
def test_a_keys_turns_stay_chronological_when_its_canonical_user_changes(self) -> None:
|
||||
client: Final = _FakeClient()
|
||||
earlier: Final = _transaction(user_id="z-user", at=datetime(2026, 8, 1, 12, 0, 0))
|
||||
later: Final = _transaction(user_id="a-user", at=datetime(2026, 8, 1, 12, 0, 10))
|
||||
asyncio.run(flush_autorouter_turn_transactions(client, [later, earlier]))
|
||||
assert [(params[5], params[19]) for _, params in client.db.calls] == [
|
||||
("2026-08-01T12:00:00", "z-user"),
|
||||
("2026-08-01T12:00:10", "a-user"),
|
||||
]
|
||||
|
||||
def test_one_keyless_users_failed_session_does_not_drop_another_users_turn(self) -> None:
|
||||
client: Final = _FakeClient(poison_user="a-user")
|
||||
failed: Final = _transaction(api_key="", user_id="a-user")
|
||||
other: Final = _transaction(api_key="", user_id="b-user", at=datetime(2026, 8, 1, 12, 0, 10))
|
||||
asyncio.run(flush_autorouter_turn_transactions(client, [other, failed]))
|
||||
assert [(params[0], params[1], params[19]) for _, params in client.db.calls] == [("", "s1", "b-user")]
|
||||
|
||||
def test_uncertain_commits_quarantine_only_the_key_and_each_failed_user(self) -> None:
|
||||
client: Final = _FakeClient(commit_then_error_users=frozenset({"a-failed", "c-failed"}))
|
||||
turns: Final = tuple(
|
||||
_transaction(user_id=user, at=datetime(2026, 8, 1, 12, 0, second), api_key=key)
|
||||
for user, second, key in (
|
||||
("b-healthy", 0, "k1"),
|
||||
("a-failed", 1, "k1"),
|
||||
("b-healthy", 2, "k1"),
|
||||
("c-failed", 3, "k1"),
|
||||
("b-healthy", 4, "k1"),
|
||||
("d-healthy", 5, "k1"),
|
||||
("c-failed", 6, "k1"),
|
||||
("d-healthy", 7, "k1"),
|
||||
("a-failed", 8, "k1"),
|
||||
("", 9, "k1"),
|
||||
("z-other", 10, "k2"),
|
||||
)
|
||||
)
|
||||
asyncio.run(flush_autorouter_turn_transactions(client, tuple(reversed(turns))))
|
||||
|
||||
assert client.db.attempts == client.db.calls
|
||||
assert [
|
||||
(params[0], params[19], params[5])
|
||||
for sql, params in client.db.calls
|
||||
if sql == UPSERT_AUTOROUTER_SESSION_SQL
|
||||
] == [
|
||||
("k1", "b-healthy", "2026-08-01T12:00:00"),
|
||||
("k1", "a-failed", "2026-08-01T12:00:01"),
|
||||
("k2", "z-other", "2026-08-01T12:00:10"),
|
||||
]
|
||||
assert [params[19] for _, params in client.db.attempts].count("a-failed") == 1
|
||||
assert [params[19] for _, params in client.db.attempts].count("c-failed") == 1
|
||||
for user, seconds in (("b-healthy", (2, 4)), ("c-failed", (3,)), ("d-healthy", (5, 7))):
|
||||
assert [
|
||||
(params[0], params[5])
|
||||
for sql, params in client.db.calls
|
||||
if sql != UPSERT_AUTOROUTER_SESSION_SQL and params[19] == user
|
||||
] == [("k1", f"2026-08-01T12:00:{second:02d}") for second in seconds]
|
||||
|
||||
def test_a_connect_error_retries_the_same_statement(self):
|
||||
client = _FakeClient(failures=[httpx.ConnectError("boom")])
|
||||
asyncio.run(flush_autorouter_turn_transactions(client, [_transaction()]))
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Unit tests for auto router management endpoints
|
|||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
|
@ -654,17 +655,43 @@ class TestAutoRouterBenchmarks:
|
|||
assert _summed_agg_row([complexity, quality]).tier_turns == {}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_admin_roles_cannot_read_benchmarks(self):
|
||||
@pytest.mark.parametrize("user_id", [None, "own-user", "other-user"])
|
||||
async def test_non_admin_roles_cannot_read_benchmarks(self, user_id: str | None):
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks
|
||||
|
||||
with pytest.raises(HTTPException) as err:
|
||||
await get_auto_router_benchmarks(
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-x"),
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-x", user_id="own-user"
|
||||
),
|
||||
start_date="2026-08-01",
|
||||
end_date="2026-08-02",
|
||||
user_id=user_id,
|
||||
)
|
||||
assert err.value.status_code == 403
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_empty_user_filter_is_rejected_before_querying_deployment_data(
|
||||
self, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
import httpx
|
||||
from fastapi import FastAPI
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks
|
||||
|
||||
query: Final = AsyncMock(return_value=[])
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", SimpleNamespace(db=SimpleNamespace(query_raw=query)))
|
||||
app: Final = FastAPI()
|
||||
app.get("/auto_router/benchmarks")(get_auto_router_benchmarks)
|
||||
app.dependency_overrides[user_api_key_auth] = lambda: ADMIN
|
||||
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client:
|
||||
response: Final = await client.get("/auto_router/benchmarks", params={"user_id": ""})
|
||||
|
||||
assert response.status_code == 422
|
||||
query.assert_not_awaited()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_reversed_window_is_rejected(self, monkeypatch: pytest.MonkeyPatch):
|
||||
from litellm.proxy import proxy_server
|
||||
|
|
@ -680,7 +707,11 @@ class TestAutoRouterBenchmarks:
|
|||
assert err.value.status_code == 400
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_returns_groups_and_totals_from_the_rollup(self, monkeypatch: pytest.MonkeyPatch):
|
||||
@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY])
|
||||
@pytest.mark.parametrize("user_id", [None, "selected-user"])
|
||||
async def test_endpoint_returns_groups_and_totals_from_the_rollup(
|
||||
self, monkeypatch: pytest.MonkeyPatch, role: LitellmUserRoles, user_id: str | None
|
||||
):
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks
|
||||
|
||||
|
|
@ -695,12 +726,13 @@ class TestAutoRouterBenchmarks:
|
|||
monkeypatch.setattr(proxy_server, "prisma_client", type("P", (), {"db": _DB()})())
|
||||
|
||||
response = await get_auto_router_benchmarks(
|
||||
user_api_key_dict=ADMIN,
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=role, api_key="sk-admin", user_id="viewer"),
|
||||
start_date="2026-07-01",
|
||||
end_date="2026-08-01",
|
||||
api_key="key-hash",
|
||||
user_id=user_id,
|
||||
)
|
||||
assert captured["params"] == ("2026-07-01T00:00:00", "2026-08-02T00:00:00", "key-hash")
|
||||
assert captured["params"] == ("2026-07-01T00:00:00", "2026-08-02T00:00:00", "key-hash", user_id)
|
||||
assert response.routers_in_scope == 1
|
||||
assert response.groups[0].router_name == "live-auto"
|
||||
assert response.groups[0].saved_pct == response.totals.saved_pct == 75.0
|
||||
|
|
|
|||
|
|
@ -793,18 +793,20 @@ async def test_spend_logs_retention_alone_does_not_touch_the_session_rollup():
|
|||
tables = [call[0][0] for call in client.db.execute_raw.call_args_list]
|
||||
assert any('"LiteLLM_SpendLogs"' in sql for sql in tables)
|
||||
assert not any('"LiteLLM_AutoRouterSession"' in sql for sql in tables)
|
||||
assert not any('"LiteLLM_AutoRouterUserSession"' in sql for sql in tables)
|
||||
assert not any('"LiteLLM_HealthCheckTable"' in sql for sql in tables)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_retention_alone_cleans_only_the_session_rollup():
|
||||
client = _mock_prisma_for_retention([0])
|
||||
async def test_session_retention_alone_cleans_both_session_rollups():
|
||||
client = _mock_prisma_for_retention([0, 0])
|
||||
cleaner = SpendLogCleanup(general_settings={"maximum_autorouter_session_retention_period": "365d"})
|
||||
cleaner.pod_lock_manager = None
|
||||
await cleaner.cleanup_old_spend_logs(client)
|
||||
tables = [call[0][0] for call in client.db.execute_raw.call_args_list]
|
||||
assert len(tables) == 1
|
||||
assert len(tables) == 2
|
||||
assert '"LiteLLM_AutoRouterSession"' in tables[0]
|
||||
assert '"LiteLLM_AutoRouterUserSession"' in tables[1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -825,7 +827,7 @@ async def test_health_check_retention_alone_cleans_only_the_health_check_table()
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_each_retention_key_cuts_off_at_its_own_horizon():
|
||||
client = _mock_prisma_for_retention([0, 0, 0, 0])
|
||||
client = _mock_prisma_for_retention([0, 0, 0, 0, 0])
|
||||
cleaner = SpendLogCleanup(
|
||||
general_settings={
|
||||
"maximum_spend_logs_retention_period": "7d",
|
||||
|
|
@ -839,6 +841,8 @@ async def test_each_retention_key_cuts_off_at_its_own_horizon():
|
|||
(
|
||||
"LiteLLM_AutoRouterSession"
|
||||
if '"LiteLLM_AutoRouterSession"' in call[0][0]
|
||||
else "LiteLLM_AutoRouterUserSession"
|
||||
if '"LiteLLM_AutoRouterUserSession"' in call[0][0]
|
||||
else "LiteLLM_HealthCheckTable"
|
||||
if '"LiteLLM_HealthCheckTable"' in call[0][0]
|
||||
else "logs"
|
||||
|
|
@ -848,6 +852,7 @@ async def test_each_retention_key_cuts_off_at_its_own_horizon():
|
|||
now = datetime.now(timezone.utc)
|
||||
assert (now - cutoffs["logs"]).days == 7
|
||||
assert (now - cutoffs["LiteLLM_AutoRouterSession"]).days == 365
|
||||
assert cutoffs["LiteLLM_AutoRouterUserSession"] == cutoffs["LiteLLM_AutoRouterSession"]
|
||||
assert (now - cutoffs["LiteLLM_HealthCheckTable"]).days == 30
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -434,7 +434,7 @@ describe("AutoRouterBenchmarksTab", () => {
|
|||
mockHook({ data: response([group()]) });
|
||||
const { dateValue, onDateChange } = renderTab();
|
||||
|
||||
expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, undefined);
|
||||
expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, undefined, undefined);
|
||||
expect(screen.getByText("Jul 6 – Aug 5 (UTC)")).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByTestId("date-picker"));
|
||||
|
|
@ -460,7 +460,7 @@ describe("AutoRouterBenchmarksTab", () => {
|
|||
</QueryClientProvider>,
|
||||
);
|
||||
|
||||
expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, "key-hash-1");
|
||||
expect(vi.mocked(useAutoRouterBenchmarks)).toHaveBeenCalledWith("sk-test", dateValue, "key-hash-1", undefined);
|
||||
expect(screen.getByText("Total estimated savings")).toBeInTheDocument();
|
||||
expect(screen.queryByRole("tab", { name: "Shadow Evals" })).not.toBeInTheDocument();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -312,8 +312,7 @@ const BenchmarksBody: React.FC<BenchmarksBodyProps> = ({ isPending, error, data,
|
|||
length. Total actual spend includes every turn; savings and baseline spend include only turns with a current
|
||||
estimate, including turns with zero savings. Savings are net of recorded LLM classification cost. Classification
|
||||
cost per 1K turns is averaged over all auto-router turns, including those that skip classification. The range
|
||||
counts whole sessions that overlap it, so totals can differ slightly from the Overall tab, which buckets savings
|
||||
by UTC day.
|
||||
counts whole sessions that overlap it, so totals can differ from savings views that group usage by UTC day.
|
||||
</p>
|
||||
|
||||
<div className="space-y-4">
|
||||
|
|
@ -333,11 +332,17 @@ interface AutoRouterBenchmarksTabProps {
|
|||
accessToken: string | null;
|
||||
activity: Pick<DailyActivityRange, "dateValue" | "onDateChange">;
|
||||
apiKey?: string;
|
||||
userId?: string;
|
||||
}
|
||||
|
||||
export const AutoRouterUsageView: React.FC<AutoRouterBenchmarksTabProps> = ({ accessToken, activity, apiKey }) => {
|
||||
export const AutoRouterUsageView: React.FC<AutoRouterBenchmarksTabProps> = ({
|
||||
accessToken,
|
||||
activity,
|
||||
apiKey,
|
||||
userId,
|
||||
}) => {
|
||||
const { dateValue, onDateChange } = activity;
|
||||
const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, dateValue, apiKey);
|
||||
const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, dateValue, apiKey, userId);
|
||||
const [selectedKey, setSelectedKey] = useState<string>(ALL_ROUTERS);
|
||||
const { data: autoRouters } = useAutoRouters();
|
||||
|
||||
|
|
@ -372,6 +377,12 @@ export const AutoRouterUsageView: React.FC<AutoRouterBenchmarksTabProps> = ({ ac
|
|||
</div>
|
||||
</div>
|
||||
|
||||
{userId && (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Usage for this user across API keys and JWT-authenticated requests. Older sessions recorded without a user ID
|
||||
are not included.
|
||||
</p>
|
||||
)}
|
||||
<BenchmarksBody
|
||||
isPending={isPending}
|
||||
error={error}
|
||||
|
|
|
|||
|
|
@ -23,10 +23,15 @@ export const benchmarksWindow = (
|
|||
};
|
||||
};
|
||||
|
||||
export const useAutoRouterBenchmarks = (accessToken: string | null, range: DateRange, apiKey?: string) =>
|
||||
export const useAutoRouterBenchmarks = (
|
||||
accessToken: string | null,
|
||||
range: DateRange,
|
||||
apiKey?: string,
|
||||
userId?: string,
|
||||
) =>
|
||||
$api.useQuery(
|
||||
"get",
|
||||
"/auto_router/benchmarks",
|
||||
{ params: { query: { ...benchmarksWindow(range, new Date()), api_key: apiKey } } },
|
||||
{ params: { query: { ...benchmarksWindow(range, new Date()), api_key: apiKey, user_id: userId } } },
|
||||
{ enabled: Boolean(accessToken && range.from && range.to), retry: false },
|
||||
);
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ vi.mock("@/app/(dashboard)/usage/_components/hooks/usePaginatedDailyActivity", (
|
|||
isFetchingMore: false,
|
||||
progress: { currentPage: 4, totalPages: 9 },
|
||||
cancelled: false,
|
||||
failed: false,
|
||||
coversRange: true,
|
||||
cancel: mockCancel,
|
||||
};
|
||||
},
|
||||
|
|
|
|||
|
|
@ -67,14 +67,16 @@ export const useScopedDailyActivityRange = (
|
|||
args: [accessToken, startTime, endTime, userId, true, apiKey],
|
||||
enabled: !!accessToken && !!startTime && !!endTime,
|
||||
};
|
||||
const { data, loading, isFetchingMore, progress, cancelled, failed, cancel } =
|
||||
const { data, loading, isFetchingMore, progress, cancelled, failed, coversRange, cancel } =
|
||||
usePaginatedDailyActivity(activityQueryOptions);
|
||||
const readUnavailable = failed || cancelled;
|
||||
const waitingForRange = activityQueryOptions.enabled && !coversRange && !readUnavailable;
|
||||
|
||||
return {
|
||||
dateValue,
|
||||
onDateChange,
|
||||
results: data.results as DailyData[],
|
||||
loading,
|
||||
loading: loading || waitingForRange,
|
||||
isFetchingMore,
|
||||
progress,
|
||||
cancelled,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,17 @@
|
|||
import { fireEvent, renderWithProviders as render, screen, waitFor } from "../../../../../../tests/test-utils";
|
||||
import {
|
||||
act,
|
||||
fireEvent,
|
||||
renderWithProviders as render,
|
||||
screen,
|
||||
testQueryClient,
|
||||
waitFor,
|
||||
} from "../../../../../../tests/test-utils";
|
||||
import userEvent, { PointerEventsCheckLevel } from "@testing-library/user-event";
|
||||
import { describe, expect, it, vi, beforeEach } from "vitest";
|
||||
import { describe, expect, it, vi, beforeEach, afterEach } from "vitest";
|
||||
import { Profiler } from "react";
|
||||
import UserInfoView from "./user_info_view";
|
||||
import type { DailyData, SpendMetrics } from "@/components/UsagePage/types";
|
||||
import type { AutoRouterBenchmarksResponse } from "@/app/(dashboard)/cost-optimization/_components/autoRouterBenchmarks";
|
||||
|
||||
const mockTeamMemberAddCall = vi.fn();
|
||||
const mockTeamMemberDeleteCall = vi.fn();
|
||||
|
|
@ -11,6 +21,8 @@ const mockTeamInfoCall = vi.fn();
|
|||
const mockUserUpdateUserCall = vi.fn();
|
||||
const mockFetchMCPServers = vi.fn();
|
||||
const mockListMCPTools = vi.fn();
|
||||
const mockUserDailyActivityCall = vi.fn();
|
||||
const mockUserDailyActivityAggregatedCall = vi.fn();
|
||||
|
||||
const MCP_SERVER = { server_id: "srv-1", server_name: "GitHub MCP", alias: "GitHub MCP" };
|
||||
|
||||
|
|
@ -47,13 +59,18 @@ vi.mock("next/navigation", () => ({
|
|||
useSearchParams: () => new URLSearchParams(window.location.search),
|
||||
}));
|
||||
|
||||
vi.mock("@/components/networking", () => {
|
||||
vi.mock("@/components/networking", async (importOriginal) => {
|
||||
const original = await importOriginal<typeof import("@/components/networking")>();
|
||||
return {
|
||||
formatDate: original.formatDate,
|
||||
serverRootPath: "/",
|
||||
userGetInfoV2: (...args: unknown[]) => mockUserGetInfoV2(...args),
|
||||
userDailyActivityCall: (...args: unknown[]) => mockUserDailyActivityCall(...args),
|
||||
userDailyActivityAggregatedCall: (...args: unknown[]) => mockUserDailyActivityAggregatedCall(...args),
|
||||
userDeleteCall: vi.fn(),
|
||||
userUpdateUserCall: (...args: unknown[]) => mockUserUpdateUserCall(...args),
|
||||
modelAvailableCall: vi.fn().mockResolvedValue({ data: [] }),
|
||||
modelInfoCall: vi.fn().mockResolvedValue({ data: [], total_pages: 1 }),
|
||||
invitationCreateCall: vi.fn(),
|
||||
teamInfoCall: (...args: unknown[]) => mockTeamInfoCall(...args),
|
||||
teamListCall: (...args: unknown[]) => mockTeamListCall(...args),
|
||||
|
|
@ -291,3 +308,337 @@ describe("UserInfoView add-to-team form", () => {
|
|||
expect(screen.getByText("Add User to Team")).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
const savingsDay = (date: string, metrics: Partial<SpendMetrics>): DailyData => ({
|
||||
date,
|
||||
metrics: {
|
||||
spend: 0,
|
||||
prompt_tokens: 0,
|
||||
completion_tokens: 0,
|
||||
total_tokens: 0,
|
||||
api_requests: 1,
|
||||
successful_requests: 1,
|
||||
failed_requests: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
cache_creation_input_tokens: 0,
|
||||
...metrics,
|
||||
},
|
||||
breakdown: { models: {}, model_groups: {}, mcp_servers: {}, providers: {}, api_keys: {}, entities: {} },
|
||||
});
|
||||
|
||||
const savingsResponse = (results: DailyData[]) => ({
|
||||
results,
|
||||
metadata: { total_pages: 1, has_more: false, page: 1 },
|
||||
});
|
||||
|
||||
const routerUsageResponse = (saved: number): AutoRouterBenchmarksResponse => ({
|
||||
start_date: "2026-09-01",
|
||||
end_date: "2026-09-19",
|
||||
routers_in_scope: 0,
|
||||
groups: [],
|
||||
totals: {
|
||||
sessions: 2,
|
||||
turns: 2,
|
||||
avg_turns_per_session: 1,
|
||||
avg_session_seconds: 0,
|
||||
avg_tokens_per_session: 100,
|
||||
spend: 10,
|
||||
savings_estimated_turns: 2,
|
||||
savings_estimated_actual_spend: 10,
|
||||
classifier_cost: 0,
|
||||
saved_spend: saved,
|
||||
baseline_spend: 10 + saved,
|
||||
saved_pct: (100 * saved) / (10 + saved),
|
||||
saved_per_session: saved / 2,
|
||||
cache: {
|
||||
coverage_pct: 100,
|
||||
hit_rate_pct: 0,
|
||||
same_model: { turns: 0, hits: 0, hit_rate_pct: 0 },
|
||||
first_visit: { turns: 2, hits: 0, hit_rate_pct: 0 },
|
||||
return_to_tier: { turns: 0, hits: 0, hit_rate_pct: 0 },
|
||||
unordered_turns: 0,
|
||||
return_misses_expired: 0,
|
||||
return_misses_within_ttl: 0,
|
||||
return_misses_unknown: 0,
|
||||
ttl_5m_turns: 0,
|
||||
ttl_1h_turns: 0,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
describe("UserInfoView auto-router usage", () => {
|
||||
const props = {
|
||||
userId: "user-123",
|
||||
onClose: vi.fn(),
|
||||
accessToken: "admin-token",
|
||||
userRole: "proxy_admin",
|
||||
possibleUIRoles: null,
|
||||
};
|
||||
const mockFetch = vi.fn<typeof fetch>();
|
||||
|
||||
beforeEach(() => {
|
||||
testQueryClient.clear();
|
||||
vi.clearAllMocks();
|
||||
mockUserGetInfoV2.mockImplementation((_token: string, userId: string) =>
|
||||
Promise.resolve({ ...MOCK_USER_DATA_NO_TEAMS, user_id: userId }),
|
||||
);
|
||||
mockFetch.mockReset().mockResolvedValue(Response.json(routerUsageResponse(42)));
|
||||
vi.stubGlobal("fetch", mockFetch);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
testQueryClient.clear();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it.each(["proxy_admin", "proxy_admin_viewer"])(
|
||||
"loads selected-user usage lazily for %s without a key filter",
|
||||
async (userRole) => {
|
||||
const user = userEvent.setup();
|
||||
render(<UserInfoView {...props} userRole={userRole} />);
|
||||
const tab = await screen.findByRole("tab", { name: "Auto-router usage" });
|
||||
expect(mockFetch).not.toHaveBeenCalled();
|
||||
await user.click(tab);
|
||||
|
||||
expect(await screen.findByText("$42.00")).toBeInTheDocument();
|
||||
const request = mockFetch.mock.calls[0][0] as Request;
|
||||
const params = new URL(request.url).searchParams;
|
||||
expect(params.get("user_id")).toBe("user-123");
|
||||
expect(params.has("api_key")).toBe(false);
|
||||
expect(screen.getByText(/Older sessions recorded without a user ID are not included/)).toBeInTheDocument();
|
||||
},
|
||||
);
|
||||
|
||||
it("switches query scope without displaying the previous user's usage", async () => {
|
||||
const nextUser = Promise.withResolvers<Response>();
|
||||
mockFetch.mockResolvedValueOnce(Response.json(routerUsageResponse(42))).mockReturnValue(nextUser.promise);
|
||||
const user = userEvent.setup();
|
||||
const { rerender } = render(<UserInfoView {...props} />);
|
||||
await user.click(await screen.findByRole("tab", { name: "Auto-router usage" }));
|
||||
expect(await screen.findByText("$42.00")).toBeInTheDocument();
|
||||
|
||||
rerender(<UserInfoView {...props} userId="user-456" />);
|
||||
expect(screen.getByText("Loading auto-router usage...")).toBeInTheDocument();
|
||||
expect(screen.queryByText("$42.00")).not.toBeInTheDocument();
|
||||
await act(async () => nextUser.resolve(Response.json(routerUsageResponse(-7))));
|
||||
expect(await screen.findByText("-$7.00")).toBeInTheDocument();
|
||||
expect(
|
||||
mockFetch.mock.calls.map(([request]) => new URL((request as Request).url).searchParams.get("user_id")),
|
||||
).toEqual(["user-123", "user-456"]);
|
||||
});
|
||||
|
||||
it.each(["internal_user", "org_admin", null])("keeps the admin-only tab unavailable to %s", async (userRole) => {
|
||||
render(<UserInfoView {...props} userRole={userRole} />);
|
||||
await screen.findByRole("tab", { name: "Overview" });
|
||||
expect(screen.queryByRole("tab", { name: "Auto-router usage" })).not.toBeInTheDocument();
|
||||
expect(mockFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("never turns an absent user ID into a deployment-wide request", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<UserInfoView {...props} userId="" />);
|
||||
await user.click(await screen.findByRole("tab", { name: "Auto-router usage" }));
|
||||
expect(screen.getByRole("alert")).toHaveTextContent("this user has no ID");
|
||||
expect(mockFetch).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("UserInfoView savings", () => {
|
||||
const props = {
|
||||
userId: "user-123",
|
||||
onClose: vi.fn(),
|
||||
accessToken: "admin-token",
|
||||
userRole: "proxy_admin",
|
||||
possibleUIRoles: null,
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockUserGetInfoV2.mockImplementation((_token: string, userId: string) =>
|
||||
Promise.resolve({ ...MOCK_USER_DATA_NO_TEAMS, user_id: userId }),
|
||||
);
|
||||
mockUserDailyActivityAggregatedCall.mockReset().mockResolvedValue(savingsResponse([]));
|
||||
mockUserDailyActivityCall.mockReset().mockResolvedValue(savingsResponse([]));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it.each(["internal_user", "org_admin", "team_admin"])(
|
||||
"only offers self savings to %s and stops querying after switching to another user",
|
||||
async (userRole) => {
|
||||
const user = userEvent.setup();
|
||||
const { rerender } = render(<UserInfoView {...props} userId="user-1" userRole={userRole} />);
|
||||
await user.click(await screen.findByRole("tab", { name: "Savings" }));
|
||||
expect(await screen.findByText("No usage recorded for this user in this range.")).toBeInTheDocument();
|
||||
expect(mockUserDailyActivityAggregatedCall.mock.calls[0][3]).toBe("user-1");
|
||||
|
||||
mockUserDailyActivityAggregatedCall.mockClear();
|
||||
mockUserDailyActivityCall.mockClear();
|
||||
rerender(<UserInfoView {...props} userId="another-user" userRole={userRole} />);
|
||||
await screen.findAllByText("another-user");
|
||||
expect(screen.getByRole("tab", { name: "Overview" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.queryByRole("tab", { name: "Savings" })).not.toBeInTheDocument();
|
||||
expect(screen.queryByText("No usage recorded for this user in this range.")).not.toBeInTheDocument();
|
||||
expect(mockUserDailyActivityAggregatedCall).not.toHaveBeenCalled();
|
||||
expect(mockUserDailyActivityCall).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("loads selected user savings without a key filter, including losses", async () => {
|
||||
const firstDay: Partial<SpendMetrics> = {
|
||||
compression_savings_spend: 1.5,
|
||||
gateway_injected_caching_savings_spend: 0.1,
|
||||
prompt_caching_savings_spend: 0.25,
|
||||
autorouter_savings_spend: -1,
|
||||
};
|
||||
const secondDay: Partial<SpendMetrics> = {
|
||||
compression_savings_spend: 0.5,
|
||||
gateway_injected_caching_savings_spend: 0.3,
|
||||
prompt_caching_savings_spend: 0.75,
|
||||
autorouter_savings_spend: -2,
|
||||
};
|
||||
mockUserDailyActivityAggregatedCall.mockResolvedValue(
|
||||
savingsResponse([savingsDay("2026-09-18", firstDay), savingsDay("2026-09-19", secondDay)]),
|
||||
);
|
||||
const user = userEvent.setup();
|
||||
render(<UserInfoView {...props} />);
|
||||
const savingsTab = await screen.findByRole("tab", { name: "Savings" });
|
||||
expect(mockUserDailyActivityAggregatedCall).not.toHaveBeenCalled();
|
||||
expect(mockUserDailyActivityCall).not.toHaveBeenCalled();
|
||||
|
||||
await user.click(savingsTab);
|
||||
|
||||
expect(await screen.findByTestId("summary-card-total-recorded-savings")).toHaveTextContent("-$0.6000");
|
||||
expect(screen.getByTestId("summary-card-compression-savings")).toHaveTextContent("$2.00");
|
||||
expect(screen.getByTestId("summary-card-prompt-caching-savings")).toHaveTextContent("$0.4000");
|
||||
expect(screen.getByTestId("summary-card-prompt-caching-savings")).toHaveTextContent("$1.00Total");
|
||||
expect(screen.getByTestId("summary-card-auto-router-savings")).toHaveTextContent("-$3.00");
|
||||
expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledExactlyOnceWith(
|
||||
"admin-token",
|
||||
expect.any(Date),
|
||||
expect.any(Date),
|
||||
"user-123",
|
||||
true,
|
||||
null,
|
||||
);
|
||||
expect(screen.getByTestId("user-savings-scope-note")).toHaveTextContent("JWT-authenticated requests");
|
||||
await user.click(screen.getByRole("tab", { name: "Per day" }));
|
||||
expect(screen.getByRole("tab", { name: "Per day" })).toHaveAttribute("aria-selected", "true");
|
||||
expect(screen.getByTestId("summary-card-total-recorded-savings")).toHaveTextContent("-$0.6000");
|
||||
});
|
||||
|
||||
it("removes the prior user's savings while the newly selected user's results are loading", async () => {
|
||||
const nextUser = Promise.withResolvers<ReturnType<typeof savingsResponse>>();
|
||||
mockUserDailyActivityAggregatedCall
|
||||
.mockResolvedValueOnce(savingsResponse([savingsDay("2026-09-19", { compression_savings_spend: 42 })]))
|
||||
.mockReturnValueOnce(nextUser.promise);
|
||||
const user = userEvent.setup();
|
||||
const { rerender } = render(<UserInfoView {...props} />);
|
||||
await user.click(await screen.findByRole("tab", { name: "Savings" }));
|
||||
expect(await screen.findByTestId("summary-card-total-recorded-savings")).toHaveTextContent("$42.00");
|
||||
|
||||
rerender(<UserInfoView {...props} userId="user-456" />);
|
||||
|
||||
expect(await screen.findByTestId("user-savings-empty")).toHaveTextContent("Loading savings");
|
||||
expect(screen.queryByTestId("summary-card-total-recorded-savings")).not.toBeInTheDocument();
|
||||
expect(mockUserDailyActivityAggregatedCall).toHaveBeenLastCalledWith(
|
||||
"admin-token",
|
||||
expect.any(Date),
|
||||
expect.any(Date),
|
||||
"user-456",
|
||||
true,
|
||||
null,
|
||||
);
|
||||
await act(async () => {
|
||||
nextUser.resolve(savingsResponse([savingsDay("2026-09-19", { autorouter_savings_spend: -7 })]));
|
||||
});
|
||||
expect(await screen.findByTestId("summary-card-total-recorded-savings")).toHaveTextContent("-$7.00");
|
||||
expect(screen.queryByText("$42.00")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("never commits the previous range's savings under the newly selected dates", async () => {
|
||||
vi.stubGlobal("requestIdleCallback", (callback: IdleRequestCallback) =>
|
||||
window.setTimeout(() => callback({ didTimeout: false, timeRemaining: () => 0 }), 0),
|
||||
);
|
||||
const nextRange = Promise.withResolvers<ReturnType<typeof savingsResponse>>();
|
||||
mockUserDailyActivityAggregatedCall
|
||||
.mockResolvedValueOnce(savingsResponse([savingsDay("2026-09-19", { compression_savings_spend: 42 })]))
|
||||
.mockReturnValue(nextRange.promise);
|
||||
const committedTotals: Array<string | null> = [];
|
||||
const captureNewRange = () => {
|
||||
if (screen.queryByText("Running total saved · Sep 1 – Sep 2 (UTC)")) {
|
||||
committedTotals.push(screen.queryByTestId("summary-card-total-recorded-savings")?.textContent ?? null);
|
||||
}
|
||||
};
|
||||
const user = userEvent.setup();
|
||||
render(
|
||||
<Profiler id="user-savings" onRender={captureNewRange}>
|
||||
<UserInfoView {...props} />
|
||||
</Profiler>,
|
||||
);
|
||||
await user.click(await screen.findByRole("tab", { name: "Savings" }));
|
||||
expect(await screen.findByTestId("summary-card-total-recorded-savings")).toHaveTextContent("$42.00");
|
||||
|
||||
await user.click(screen.getByRole("button", { name: / - / }));
|
||||
const [startDateInput, endDateInput] = screen.getAllByDisplayValue(/^\d{4}-\d{2}-\d{2}$/);
|
||||
fireEvent.change(startDateInput, { target: { value: "2026-09-01" } });
|
||||
fireEvent.change(endDateInput, { target: { value: "2026-09-02" } });
|
||||
await user.click(screen.getByRole("button", { name: "Apply" }));
|
||||
|
||||
expect(committedTotals.length).toBeGreaterThan(0);
|
||||
expect(committedTotals.every((total) => total === null)).toBe(true);
|
||||
expect(screen.getByTestId("user-savings-empty")).toHaveTextContent("Loading savings");
|
||||
await act(async () => {
|
||||
nextRange.resolve(savingsResponse([savingsDay("2026-09-02", { autorouter_savings_spend: -7 })]));
|
||||
});
|
||||
expect(await screen.findByTestId("summary-card-total-recorded-savings")).toHaveTextContent("-$7.00");
|
||||
});
|
||||
|
||||
it("reports an incomplete paginated read as unavailable instead of displaying a partial savings total", async () => {
|
||||
mockUserDailyActivityAggregatedCall.mockRejectedValue(new Error("aggregated unavailable"));
|
||||
mockUserDailyActivityCall
|
||||
.mockResolvedValueOnce({
|
||||
results: [savingsDay("2026-09-19", { compression_savings_spend: 42 })],
|
||||
metadata: { total_pages: 2, has_more: true, page: 1 },
|
||||
})
|
||||
.mockRejectedValueOnce(new Error("next page unavailable"));
|
||||
const user = userEvent.setup();
|
||||
render(<UserInfoView {...props} />);
|
||||
await user.click(await screen.findByRole("tab", { name: "Savings" }));
|
||||
|
||||
expect(await screen.findByRole("alert")).toHaveTextContent("Savings are unavailable for this range");
|
||||
expect(mockUserDailyActivityCall).toHaveBeenLastCalledWith(
|
||||
"admin-token",
|
||||
expect.any(Date),
|
||||
expect.any(Date),
|
||||
2,
|
||||
"user-123",
|
||||
true,
|
||||
null,
|
||||
);
|
||||
expect(screen.queryByTestId("summary-card-total-recorded-savings")).not.toBeInTheDocument();
|
||||
expect(screen.queryByText(/No usage recorded/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("distinguishes a user with no usage from an unavailable read", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<UserInfoView {...props} />);
|
||||
await user.click(await screen.findByRole("tab", { name: "Savings" }));
|
||||
|
||||
expect(await screen.findByTestId("user-savings-empty")).toHaveTextContent("No usage recorded for this user");
|
||||
expect(screen.getByTestId("summary-card-total-recorded-savings")).toHaveTextContent("$0.00");
|
||||
expect(screen.queryByRole("alert")).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it.each(["", " "])("never queries an absent selected user ID (%j)", async (userId) => {
|
||||
const user = userEvent.setup();
|
||||
render(<UserInfoView {...props} userId={userId} />);
|
||||
await user.click(await screen.findByRole("tab", { name: "Savings" }));
|
||||
|
||||
expect(screen.getByRole("alert")).toHaveTextContent("this user has no ID");
|
||||
expect(mockUserDailyActivityAggregatedCall).not.toHaveBeenCalled();
|
||||
expect(mockUserDailyActivityCall).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import {
|
|||
ComboboxList,
|
||||
} from "@/components/ui/combobox";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { rolesWithWriteAccess } from "@/utils/roles";
|
||||
import { hasProxyWideSpendView, rolesWithWriteAccess } from "@/utils/roles";
|
||||
import { teamDetailHref } from "@/utils/entityLinks";
|
||||
import { BadgeLink } from "@/components/shared/BadgeLink";
|
||||
import { UserEditView } from "../user_edit_view";
|
||||
|
|
@ -44,6 +44,9 @@ import { useMCPServers } from "@/app/(dashboard)/hooks/mcpServers/useMCPServers"
|
|||
import { useMCPToolsets } from "@/app/(dashboard)/hooks/mcpServers/useMCPToolsets";
|
||||
import { extractMcpEntitlement } from "@/components/mcp_server_management/mcpEntitlement";
|
||||
import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog";
|
||||
import ScopedSavingsTab from "@/components/shared/ScopedSavingsTab";
|
||||
import { AutoRouterUsageView } from "@/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab";
|
||||
import { useActivityDateRange } from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange";
|
||||
|
||||
interface UserInfoViewProps {
|
||||
userId: string;
|
||||
|
|
@ -85,7 +88,10 @@ export default function UserInfoView({
|
|||
initialTab = 0,
|
||||
startInEditMode = false,
|
||||
}: UserInfoViewProps) {
|
||||
const { premiumUser } = useAuthorized();
|
||||
const { premiumUser, userId: signedInUserId } = useAuthorized();
|
||||
const canViewAutoRouterUsage = hasProxyWideSpendView(userRole);
|
||||
const canViewSavings = canViewAutoRouterUsage || (Boolean(userId.trim()) && userId === signedInUserId);
|
||||
const activityDateRange = useActivityDateRange();
|
||||
const [userData, setUserData] = useState<UserInfoV2Response | null>(null);
|
||||
const [teamDetails, setTeamDetails] = useState<TeamDisplayInfo[]>([]);
|
||||
const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
|
||||
|
|
@ -97,6 +103,8 @@ export default function UserInfoView({
|
|||
const [invitationLinkData, setInvitationLinkData] = useState<InvitationLink | null>(null);
|
||||
const [baseUrl, setBaseUrl] = useState<string | null>(null);
|
||||
const [activeTab, setActiveTab] = useState<string>(initialTab === 1 ? "details" : "overview");
|
||||
const hiddenSavingsTab = activeTab === "savings" && !canViewSavings;
|
||||
const hiddenRouterTab = activeTab === "auto-router-usage" && !canViewAutoRouterUsage;
|
||||
const [copiedStates, setCopiedStates] = useState<Record<string, boolean>>({});
|
||||
const [isTeamsExpanded, setIsTeamsExpanded] = useState(false);
|
||||
const [isAddTeamModalOpen, setIsAddTeamModalOpen] = useState(false);
|
||||
|
|
@ -467,7 +475,11 @@ export default function UserInfoView({
|
|||
confirmLoading={isDeletingUser}
|
||||
/>
|
||||
|
||||
<Tabs value={activeTab} onValueChange={(v: unknown) => setActiveTab(String(v))} className="gap-0">
|
||||
<Tabs
|
||||
value={hiddenSavingsTab || hiddenRouterTab ? "overview" : activeTab}
|
||||
onValueChange={(v: unknown) => setActiveTab(String(v))}
|
||||
className="gap-0"
|
||||
>
|
||||
<TabsList variant="line" className="mb-4">
|
||||
<TabsTrigger value="overview" className="flex-none data-active:text-primary after:bg-primary">
|
||||
Overview
|
||||
|
|
@ -475,6 +487,16 @@ export default function UserInfoView({
|
|||
<TabsTrigger value="details" className="flex-none data-active:text-primary after:bg-primary">
|
||||
Details
|
||||
</TabsTrigger>
|
||||
{canViewSavings && (
|
||||
<TabsTrigger value="savings" className="flex-none data-active:text-primary after:bg-primary">
|
||||
Savings
|
||||
</TabsTrigger>
|
||||
)}
|
||||
{canViewAutoRouterUsage && (
|
||||
<TabsTrigger value="auto-router-usage" className="flex-none data-active:text-primary after:bg-primary">
|
||||
Auto-router usage
|
||||
</TabsTrigger>
|
||||
)}
|
||||
</TabsList>
|
||||
|
||||
{/* Overview Panel */}
|
||||
|
|
@ -685,6 +707,38 @@ export default function UserInfoView({
|
|||
)}
|
||||
</Card>
|
||||
</TabsContent>
|
||||
{canViewSavings && (
|
||||
<TabsContent value="savings">
|
||||
{activeTab === "savings" &&
|
||||
(userId.trim() ? (
|
||||
<ScopedSavingsTab
|
||||
key={userId}
|
||||
accessToken={accessToken}
|
||||
scope={{ userId }}
|
||||
activity={activityDateRange}
|
||||
entityType="user"
|
||||
scopeNote="Savings for this user across API keys and JWT-authenticated requests."
|
||||
/>
|
||||
) : (
|
||||
<p role="alert">Savings are unavailable because this user has no ID.</p>
|
||||
))}
|
||||
</TabsContent>
|
||||
)}
|
||||
{canViewAutoRouterUsage && (
|
||||
<TabsContent value="auto-router-usage">
|
||||
{activeTab === "auto-router-usage" &&
|
||||
(userId.trim() ? (
|
||||
<AutoRouterUsageView
|
||||
key={userId}
|
||||
accessToken={accessToken}
|
||||
userId={userId}
|
||||
activity={activityDateRange}
|
||||
/>
|
||||
) : (
|
||||
<p role="alert">Auto-router usage is unavailable because this user has no ID.</p>
|
||||
))}
|
||||
</TabsContent>
|
||||
)}
|
||||
</Tabs>
|
||||
<OnboardingModal
|
||||
isInvitationLinkModalVisible={isInvitationLinkModalVisible}
|
||||
|
|
|
|||
133
ui/litellm-dashboard/src/components/shared/ScopedSavingsTab.tsx
Normal file
133
ui/litellm-dashboard/src/components/shared/ScopedSavingsTab.tsx
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
"use client";
|
||||
|
||||
import { useMemo, useState } from "react";
|
||||
|
||||
import { AreaChart, BarChart, CustomLegend } from "@/components/shared/charts";
|
||||
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
|
||||
import SavingsTiles from "@/components/shared/SavingsTiles";
|
||||
import { Card, CardAction, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import {
|
||||
formatRangeLabel,
|
||||
localIsoDay,
|
||||
MAX_POINTS_WITH_DOTS,
|
||||
SAVINGS_COLORS,
|
||||
SAVINGS_SERIES,
|
||||
SavingsAccumulation,
|
||||
SavingsPoint,
|
||||
savingsSeriesOf,
|
||||
shortDate,
|
||||
toCumulative,
|
||||
usd,
|
||||
withStartAnchor,
|
||||
} from "@/app/(dashboard)/cost-optimization/_components/costOptimizationUtils";
|
||||
import {
|
||||
useScopedDailyActivityRange,
|
||||
type ActivityDateRange,
|
||||
type DailyActivityScope,
|
||||
} from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange";
|
||||
|
||||
interface ScopedSavingsTabProps {
|
||||
accessToken: string | null;
|
||||
scope: DailyActivityScope;
|
||||
activity: ActivityDateRange;
|
||||
entityType: "key" | "user";
|
||||
scopeNote?: string;
|
||||
}
|
||||
|
||||
const ScopedSavingsTab = ({ accessToken, scope, activity, entityType, scopeNote }: ScopedSavingsTabProps) => {
|
||||
const { dateValue, onDateChange, results, loading, isFetchingMore, failed, cancelled } = useScopedDailyActivityRange(
|
||||
accessToken,
|
||||
scope,
|
||||
activity,
|
||||
);
|
||||
const startTime = dateValue.from;
|
||||
const endTime = dateValue.to;
|
||||
|
||||
const [accumulation, setAccumulation] = useState<SavingsAccumulation>("cumulative");
|
||||
|
||||
const perInterval = useMemo<SavingsPoint[]>(() => savingsSeriesOf(results), [results]);
|
||||
|
||||
const overTime = useMemo(() => {
|
||||
if (accumulation !== "cumulative") return perInterval;
|
||||
const startLabel = startTime ? shortDate(localIsoDay(startTime)) : "";
|
||||
return withStartAnchor(toCumulative(perInterval), startLabel);
|
||||
}, [accumulation, perInterval, startTime]);
|
||||
|
||||
const intervalLabel = "Per day";
|
||||
const rangeLabel = formatRangeLabel(startTime, endTime);
|
||||
const savingsSubtitle = [
|
||||
accumulation === "cumulative" ? "Running total saved" : `Saved ${intervalLabel.toLowerCase()}`,
|
||||
rangeLabel && `${rangeLabel} (UTC)`,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" · ");
|
||||
|
||||
const isLoading = loading || isFetchingMore;
|
||||
const unavailable = failed || cancelled;
|
||||
const showResults = !isLoading && !unavailable;
|
||||
const hasRows = results.length > 0;
|
||||
const showEmpty = !unavailable && (isLoading || !hasRows);
|
||||
const showChart = showResults && hasRows;
|
||||
const chartProps = {
|
||||
data: overTime,
|
||||
index: "date",
|
||||
categories: SAVINGS_SERIES,
|
||||
colors: SAVINGS_COLORS,
|
||||
valueFormatter: usd,
|
||||
showLegend: false,
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="w-full space-y-6">
|
||||
<div className="flex flex-wrap items-center justify-end gap-4">
|
||||
<span className="text-sm text-muted-foreground">Spend is bucketed by UTC day</span>
|
||||
<AdvancedDatePicker value={dateValue} onValueChange={onDateChange} />
|
||||
</div>
|
||||
|
||||
{scopeNote && (
|
||||
<p className="text-sm text-muted-foreground" data-testid={`${entityType}-savings-scope-note`}>
|
||||
{scopeNote}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{unavailable && (
|
||||
<p role="alert" className="text-sm text-muted-foreground">
|
||||
Savings are unavailable for this range. Try another date range or reopen this tab.
|
||||
</p>
|
||||
)}
|
||||
{showResults && <SavingsTiles results={results} isLoading={false} />}
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Savings</CardTitle>
|
||||
<CardDescription>{savingsSubtitle}</CardDescription>
|
||||
<CardAction className="flex flex-wrap items-center justify-end gap-x-4 gap-y-2">
|
||||
<CustomLegend categories={SAVINGS_SERIES} colors={SAVINGS_COLORS} />
|
||||
<Tabs value={accumulation} onValueChange={(value) => setAccumulation(value as SavingsAccumulation)}>
|
||||
<TabsList>
|
||||
<TabsTrigger value="cumulative">Cumulative</TabsTrigger>
|
||||
<TabsTrigger value="per-interval">{intervalLabel}</TabsTrigger>
|
||||
</TabsList>
|
||||
</Tabs>
|
||||
</CardAction>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
{showEmpty && (
|
||||
<p className="py-12 text-center text-sm text-muted-foreground" data-testid={`${entityType}-savings-empty`}>
|
||||
{isLoading ? "Loading savings..." : `No usage recorded for this ${entityType} in this range.`}
|
||||
</p>
|
||||
)}
|
||||
{showChart &&
|
||||
(accumulation === "cumulative" ? (
|
||||
<AreaChart {...chartProps} showDots={overTime.length <= MAX_POINTS_WITH_DOTS} />
|
||||
) : (
|
||||
<BarChart {...chartProps} />
|
||||
))}
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default ScopedSavingsTab;
|
||||
|
|
@ -1,132 +1,29 @@
|
|||
"use client";
|
||||
|
||||
import React, { useMemo, useState } from "react";
|
||||
|
||||
import { AreaChart, BarChart, CustomLegend } from "@/components/shared/charts";
|
||||
import AdvancedDatePicker from "@/components/shared/advanced_date_picker";
|
||||
import SavingsTiles from "@/components/shared/SavingsTiles";
|
||||
import { Card, CardAction, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
||||
import ScopedSavingsTab from "@/components/shared/ScopedSavingsTab";
|
||||
import { hasProxyWideSpendView, spendScopeUserId } from "@/utils/roles";
|
||||
import {
|
||||
formatRangeLabel,
|
||||
localIsoDay,
|
||||
MAX_POINTS_WITH_DOTS,
|
||||
SAVINGS_COLORS,
|
||||
SAVINGS_SERIES,
|
||||
SavingsAccumulation,
|
||||
SavingsPoint,
|
||||
savingsSeriesOf,
|
||||
shortDate,
|
||||
toCumulative,
|
||||
usd,
|
||||
withStartAnchor,
|
||||
} from "@/app/(dashboard)/cost-optimization/_components/costOptimizationUtils";
|
||||
import {
|
||||
useScopedDailyActivityRange,
|
||||
type ActivityDateRange,
|
||||
} from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange";
|
||||
import type { ActivityDateRange } from "@/app/(dashboard)/cost-optimization/_components/useDailyActivityRange";
|
||||
|
||||
interface KeySavingsTabProps {
|
||||
accessToken: string | null;
|
||||
/** The key's token hash — what spend rows are keyed by, not the one-time plaintext secret. */
|
||||
keyToken: string;
|
||||
userId: string | null;
|
||||
userRole: string;
|
||||
activity: ActivityDateRange;
|
||||
}
|
||||
|
||||
const KeySavingsTab: React.FC<KeySavingsTabProps> = ({ accessToken, keyToken, userId, userRole, activity }) => {
|
||||
// Proxy admins read the whole key. For anyone else the endpoint applies the caller's own user_id
|
||||
// alongside the key filter, so the figures cover only that viewer's requests on this key -- said
|
||||
// plainly in the scope note below rather than left to be misread as the key's total.
|
||||
const readsWholeKey = hasProxyWideSpendView(userRole);
|
||||
const { dateValue, onDateChange, results, loading, isFetchingMore } = useScopedDailyActivityRange(
|
||||
accessToken,
|
||||
{ userId: spendScopeUserId(userRole, userId), apiKey: keyToken },
|
||||
activity,
|
||||
);
|
||||
const startTime = dateValue.from ?? null;
|
||||
const endTime = dateValue.to ?? null;
|
||||
|
||||
const [accumulation, setAccumulation] = useState<SavingsAccumulation>("cumulative");
|
||||
|
||||
const perInterval = useMemo<SavingsPoint[]>(() => savingsSeriesOf(results), [results]);
|
||||
|
||||
const overTime = useMemo(() => {
|
||||
if (accumulation !== "cumulative") return perInterval;
|
||||
const startLabel = startTime ? shortDate(localIsoDay(startTime)) : "";
|
||||
return withStartAnchor(toCumulative(perInterval), startLabel);
|
||||
}, [accumulation, perInterval, startTime]);
|
||||
|
||||
const intervalLabel = "Per day";
|
||||
const rangeLabel = formatRangeLabel(startTime ?? undefined, endTime ?? undefined);
|
||||
const savingsSubtitle = [
|
||||
accumulation === "cumulative" ? "Running total saved" : `Saved ${intervalLabel.toLowerCase()}`,
|
||||
rangeLabel && `${rangeLabel} (UTC)`,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" · ");
|
||||
|
||||
const isLoading = loading || isFetchingMore;
|
||||
const hasRows = results.length > 0;
|
||||
const chartProps = {
|
||||
data: overTime,
|
||||
index: "date",
|
||||
categories: SAVINGS_SERIES,
|
||||
colors: SAVINGS_COLORS,
|
||||
valueFormatter: usd,
|
||||
showLegend: false,
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="w-full space-y-6">
|
||||
<div className="flex flex-wrap items-center justify-end gap-4">
|
||||
<span className="text-sm text-muted-foreground">Spend is bucketed by UTC day</span>
|
||||
<AdvancedDatePicker value={dateValue} onValueChange={onDateChange} />
|
||||
</div>
|
||||
|
||||
{!readsWholeKey && (
|
||||
<p className="text-sm text-muted-foreground" data-testid="key-savings-scope-note">
|
||||
Showing your own requests on this key. A key shared across a team will have spend from other members that is
|
||||
not counted here.
|
||||
</p>
|
||||
)}
|
||||
|
||||
<SavingsTiles results={results} isLoading={isLoading} />
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Savings</CardTitle>
|
||||
<CardDescription>{savingsSubtitle}</CardDescription>
|
||||
<CardAction className="flex flex-wrap items-center justify-end gap-x-4 gap-y-2">
|
||||
<CustomLegend categories={SAVINGS_SERIES} colors={SAVINGS_COLORS} />
|
||||
<Tabs value={accumulation} onValueChange={(value) => setAccumulation(value as SavingsAccumulation)}>
|
||||
<TabsList>
|
||||
<TabsTrigger value="cumulative">Cumulative</TabsTrigger>
|
||||
<TabsTrigger value="per-interval">{intervalLabel}</TabsTrigger>
|
||||
</TabsList>
|
||||
</Tabs>
|
||||
</CardAction>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
{/* Distinguishes "still fetching" from "this key genuinely had no traffic": an empty
|
||||
chart alone reads as a broken panel, and a $0.00 tile reads as a real zero. */}
|
||||
{!hasRows && (
|
||||
<p className="py-12 text-center text-sm text-muted-foreground" data-testid="key-savings-empty">
|
||||
{isLoading ? "Loading savings..." : "No usage recorded for this key in this range."}
|
||||
</p>
|
||||
)}
|
||||
{hasRows && accumulation === "cumulative" && (
|
||||
<AreaChart {...chartProps} showDots={overTime.length <= MAX_POINTS_WITH_DOTS} />
|
||||
)}
|
||||
{/* Not stacked: auto-router can go negative on a cold-cache write, and stacking would
|
||||
draw that below the axis while the rest of the bar still read as the total. */}
|
||||
{hasRows && accumulation !== "cumulative" && <BarChart {...chartProps} />}
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
const KeySavingsTab = ({ accessToken, keyToken, userId, userRole, activity }: KeySavingsTabProps) => (
|
||||
<ScopedSavingsTab
|
||||
accessToken={accessToken}
|
||||
scope={{ userId: spendScopeUserId(userRole, userId), apiKey: keyToken }}
|
||||
activity={activity}
|
||||
entityType="key"
|
||||
scopeNote={
|
||||
hasProxyWideSpendView(userRole)
|
||||
? undefined
|
||||
: "Showing your own requests on this key. A key shared across a team will have spend from other members that is not counted here."
|
||||
}
|
||||
/>
|
||||
);
|
||||
|
||||
export default KeySavingsTab;
|
||||
|
|
|
|||
9
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
9
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -1246,9 +1246,10 @@ export interface paths {
|
|||
* @description Benchmarks for the auto-router dashboard: session shape, savings against the configured
|
||||
* baseline, and prompt-caching behaviour bucketed by what the router did.
|
||||
*
|
||||
* Reads the LiteLLM_AutoRouterSession rollup, folded once per request at spend-write time,
|
||||
* so this endpoint never scans LiteLLM_SpendLogs. A session is in the window when it
|
||||
* overlaps it: its last turn is on or after start_date and its first turn is on or before
|
||||
* Reads session rollups folded once per request at spend-write time, so this endpoint
|
||||
* never scans LiteLLM_SpendLogs. A user filter selects only turns attributed to that
|
||||
* internal user when written; older key-only history remains outside user views. A session
|
||||
* is in the window when it overlaps it: its last turn is on or after start_date and its first turn is on or before
|
||||
* end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is
|
||||
* over that bucket's turns.
|
||||
*
|
||||
|
|
@ -43601,6 +43602,8 @@ export interface operations {
|
|||
end_date?: string | null;
|
||||
/** @description Filter to one virtual key token hash */
|
||||
api_key?: string | null;
|
||||
/** @description Filter to one canonical internal user recorded on each turn */
|
||||
user_id?: string | null;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue