mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
feat(proxy): add auto-router session benchmarks endpoint and UI
Adds GET /auto_router/benchmarks, an admin-only read over spend logs that reports, per configured auto-router, the four session metrics customers ask for (turns per session, average session length, tokens per session) plus a routed-vs-baseline dollar savings estimate. Session-scoped metrics can't come from the daily rollups because session_id lives only on LiteLLM_SpendLogs, so this reads spend logs directly over a window clamped to 30 days, matching the tool-spend endpoint. Spend is filtered by the auto-router alias, which keeps the turn count honest: the LLM classifier's own judge calls share the session but carry the judge model's group, not the alias. The savings figure compares the routed model mix against sending every request to a single baseline model, defaulting to the priciest model the router actually routed to in the window. A per-router benchmark_baseline_model param pins it to a fixed flagship instead. The estimate uses list prices on the tokens actually spent and does not model the caching a single-model baseline would have had; the UI says so. Surfaced as stat tiles on the Cost Optimization autorouter tab.
This commit is contained in:
Tin Chi Lo
parent
04177e7419
commit
703fde9f1e
11 changed files with 949 additions and 1 deletions
|
|
@ -19,6 +19,7 @@ from collections.abc import Mapping
|
|||
from datetime import datetime, timedelta, timezone
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Annotated,
|
||||
Any,
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
|
|
@ -16322,6 +16323,52 @@ async def get_adaptive_router_state(
|
|||
return {"routers": snapshots}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/auto_router/benchmarks",
|
||||
tags=["auto_router"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def get_auto_router_benchmarks(
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
):
|
||||
"""Session-level benchmarks for every configured auto-router.
|
||||
|
||||
Admin-only. For each auto-router alias, reports turns per session, session
|
||||
length, tokens per session, and the dollar savings of the routed model mix
|
||||
versus sending every request to a single baseline model over the window.
|
||||
|
||||
``start_date`` / ``end_date`` are ``YYYY-MM-DD``; the window is clamped to
|
||||
the most recent ``BENCHMARKS_MAX_WINDOW_DAYS`` days and the response echoes
|
||||
the window actually served. Returns 404 when no auto-router is configured.
|
||||
"""
|
||||
from litellm.proxy.spend_tracking.auto_router_benchmarks import auto_router_groups, compute_benchmarks
|
||||
|
||||
if not _user_has_admin_view(user_api_key_dict):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={"error": CommonProxyErrors.not_allowed_access.value},
|
||||
)
|
||||
if llm_router is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"error": "No auto_router is configured on this proxy."},
|
||||
)
|
||||
groups = auto_router_groups(llm_router)
|
||||
if not groups:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"error": "No auto_router is configured on this proxy."},
|
||||
)
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": CommonProxyErrors.db_not_connected_error.value},
|
||||
)
|
||||
return await compute_benchmarks(prisma_client, groups, start_date, end_date)
|
||||
|
||||
|
||||
@router.get("/routes", dependencies=[Depends(user_api_key_auth)])
|
||||
async def get_routes():
|
||||
"""
|
||||
|
|
|
|||
333
litellm/proxy/spend_tracking/auto_router_benchmarks.py
Normal file
333
litellm/proxy/spend_tracking/auto_router_benchmarks.py
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
"""Session-level benchmarks for auto-router deployments.
|
||||
|
||||
Answers the customer question "what is the auto-router actually buying me": how
|
||||
many turns a routed session runs, how long it lasts, how many tokens it burns,
|
||||
and how much cheaper the routed model mix is than sending every request to a
|
||||
single baseline model.
|
||||
|
||||
The four metrics are session-scoped, and ``session_id`` lives only on
|
||||
``LiteLLM_SpendLogs`` (the per-request table), never on the daily rollups: a
|
||||
session does not close on a day boundary, so it cannot be pre-aggregated the way
|
||||
user/team spend is. This reads SpendLogs directly over a bounded window; the
|
||||
durable answer is a per-session rollup table, tracked as a follow-up.
|
||||
|
||||
Only rows whose ``model_group`` is a configured auto-router alias are counted.
|
||||
That filter is load-bearing: the auto-router's own LLM-classifier sub-calls land
|
||||
in the same session but carry the judge model's group, not the alias, so
|
||||
grouping by the alias yields one row per routed turn with no classifier noise
|
||||
and no separate ``call_type`` filter.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import TYPE_CHECKING, Mapping, NamedTuple
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.router import Router
|
||||
|
||||
|
||||
BENCHMARKS_MAX_WINDOW_DAYS = 30
|
||||
|
||||
|
||||
class AutoRouterGroupBenchmark(BaseModel):
|
||||
model_group: str
|
||||
router_kind: str
|
||||
baseline_model: str
|
||||
sessions: int
|
||||
turns: int
|
||||
avg_turns_per_session: float
|
||||
avg_session_length_seconds: float
|
||||
total_tokens: int
|
||||
avg_tokens_per_session: float
|
||||
actual_spend: float
|
||||
baseline_spend: float
|
||||
savings: float
|
||||
savings_pct: float
|
||||
|
||||
|
||||
class AutoRouterBenchmarksResponse(BaseModel):
|
||||
start_date: str
|
||||
end_date: str
|
||||
groups: tuple[AutoRouterGroupBenchmark, ...]
|
||||
|
||||
|
||||
class _BaselineRates(NamedTuple):
|
||||
model: str
|
||||
input_cost_per_token: float
|
||||
output_cost_per_token: float
|
||||
|
||||
|
||||
class _SessionRow(NamedTuple):
|
||||
session_id: str
|
||||
turns: int
|
||||
session_length_seconds: float
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
total_tokens: int
|
||||
actual_spend: float
|
||||
|
||||
|
||||
def auto_router_groups(router: "Router") -> tuple[tuple[str, str, str | None], ...]:
|
||||
"""Enumerate ``(public model_group, router kind, configured baseline)`` per auto-router.
|
||||
|
||||
``model_group`` is the public alias clients send and SpendLogs records; the
|
||||
``litellm_params.model`` string is the ``auto_router/...`` discriminator. A
|
||||
per-router ``benchmark_baseline_model`` override, when set, pins the baseline
|
||||
to a fixed flagship; otherwise the caller derives it from the routed traffic.
|
||||
"""
|
||||
return tuple(
|
||||
(
|
||||
str(entry["model_name"]),
|
||||
kind,
|
||||
_configured_baseline(entry.get("litellm_params")),
|
||||
)
|
||||
for entry in (router.model_list or [])
|
||||
if (model := _entry_model(entry)) is not None and (kind := classify_strategy_router_model(model)) is not None
|
||||
)
|
||||
|
||||
|
||||
def _entry_model(entry: Mapping[str, object]) -> str | None:
|
||||
params = entry.get("litellm_params")
|
||||
if not isinstance(params, Mapping):
|
||||
return None
|
||||
model = params.get("model")
|
||||
return model if isinstance(model, str) else None
|
||||
|
||||
|
||||
def _configured_baseline(params: object) -> str | None:
|
||||
if not isinstance(params, Mapping):
|
||||
return None
|
||||
baseline = params.get("benchmark_baseline_model")
|
||||
return baseline if isinstance(baseline, str) and baseline else None
|
||||
|
||||
|
||||
def _baseline_rates(model: str) -> _BaselineRates:
|
||||
"""Price a baseline model, trying provider-prefixed and bare name candidates.
|
||||
|
||||
Daily/spend rows store models provider-prefixed (``anthropic/claude-opus-5``)
|
||||
while the cost map often keys them bare, so a single lookup silently prices
|
||||
zero. Falls open to zero rates for a fully unknown model, which surfaces as a
|
||||
zero baseline (and zero savings) rather than a raised error.
|
||||
"""
|
||||
for candidate in _pricing_candidates(model):
|
||||
try:
|
||||
info = litellm.get_model_info(model=candidate)
|
||||
except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models
|
||||
verbose_proxy_logger.debug("auto_router_benchmarks: no model info for %s (%s)", candidate, e)
|
||||
continue
|
||||
return _BaselineRates(
|
||||
model=model,
|
||||
input_cost_per_token=float(info.get("input_cost_per_token") or 0.0),
|
||||
output_cost_per_token=float(info.get("output_cost_per_token") or 0.0),
|
||||
)
|
||||
verbose_proxy_logger.warning(
|
||||
"auto_router_benchmarks: baseline model %s is not priced; savings will read zero", model
|
||||
)
|
||||
return _BaselineRates(model=model, input_cost_per_token=0.0, output_cost_per_token=0.0)
|
||||
|
||||
|
||||
def _pricing_candidates(model: str) -> tuple[str, ...]:
|
||||
stripped = model.split("/", 1)[1] if "/" in model else model
|
||||
return tuple(dict.fromkeys((model, stripped)))
|
||||
|
||||
|
||||
def _clamp_window(start_date: str, end_date: str) -> tuple[datetime, datetime, str, str]:
|
||||
"""Parse the range and enforce ``start >= end - BENCHMARKS_MAX_WINDOW_DAYS``.
|
||||
|
||||
The scan is over the unbounded per-request table, so the window is capped the
|
||||
way the tool-spend endpoint caps it; the returned start reflects the window
|
||||
actually served.
|
||||
"""
|
||||
start = datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc)
|
||||
end = datetime.fromisoformat(end_date).replace(tzinfo=timezone.utc)
|
||||
floor = (end - timedelta(days=BENCHMARKS_MAX_WINDOW_DAYS)).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
clamped_start = max(start, floor)
|
||||
return clamped_start, end, clamped_start.date().isoformat(), end.date().isoformat()
|
||||
|
||||
|
||||
def _derive_baseline_model(rows: tuple["_RoutedModelSpend", ...]) -> str | None:
|
||||
"""Pick the priciest model actually routed to in the window as the baseline.
|
||||
|
||||
Uniform across all four router types and always a model the router really
|
||||
used, so "vs baseline" is grounded in the deployment rather than a guess at
|
||||
which tier is the flagship. Priced by blended per-token rate so a model is
|
||||
not called the flagship purely because it emitted more tokens.
|
||||
"""
|
||||
priced = tuple((row.model, rate) for row in rows if (rate := _blended_rate(row.model)) is not None)
|
||||
if not priced:
|
||||
return None
|
||||
return max(priced, key=lambda pair: pair[1])[0]
|
||||
|
||||
|
||||
def _blended_rate(model: str) -> float | None:
|
||||
for candidate in _pricing_candidates(model):
|
||||
try:
|
||||
info = litellm.get_model_info(model=candidate)
|
||||
except Exception: # noqa: BLE001 # unmapped model, try next candidate
|
||||
continue
|
||||
return float(info.get("input_cost_per_token") or 0.0) + float(info.get("output_cost_per_token") or 0.0)
|
||||
return None
|
||||
|
||||
|
||||
class _RoutedModelSpend(NamedTuple):
|
||||
model: str
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
|
||||
|
||||
def summarize_group(
|
||||
model_group: str,
|
||||
router_kind: str,
|
||||
session_rows: tuple[_SessionRow, ...],
|
||||
routed_models: tuple[_RoutedModelSpend, ...],
|
||||
configured_baseline: str | None,
|
||||
) -> AutoRouterGroupBenchmark | None:
|
||||
"""Fold per-session and per-model rows into one group benchmark.
|
||||
|
||||
Returns None when the window holds no routed sessions for the group, so an
|
||||
idle auto-router is omitted rather than reported as a row of zeros.
|
||||
"""
|
||||
if not session_rows:
|
||||
return None
|
||||
baseline_model = configured_baseline or _derive_baseline_model(routed_models)
|
||||
if baseline_model is None:
|
||||
return None
|
||||
rates = _baseline_rates(baseline_model)
|
||||
|
||||
sessions = len(session_rows)
|
||||
turns = sum(row.turns for row in session_rows)
|
||||
total_tokens = sum(row.total_tokens for row in session_rows)
|
||||
actual_spend = sum(row.actual_spend for row in session_rows)
|
||||
baseline_spend = sum(
|
||||
row.prompt_tokens * rates.input_cost_per_token + row.completion_tokens * rates.output_cost_per_token
|
||||
for row in session_rows
|
||||
)
|
||||
savings = baseline_spend - actual_spend
|
||||
return AutoRouterGroupBenchmark(
|
||||
model_group=model_group,
|
||||
router_kind=router_kind,
|
||||
baseline_model=baseline_model,
|
||||
sessions=sessions,
|
||||
turns=turns,
|
||||
avg_turns_per_session=turns / sessions,
|
||||
avg_session_length_seconds=sum(row.session_length_seconds for row in session_rows) / sessions,
|
||||
total_tokens=total_tokens,
|
||||
avg_tokens_per_session=total_tokens / sessions,
|
||||
actual_spend=actual_spend,
|
||||
baseline_spend=baseline_spend,
|
||||
savings=savings,
|
||||
savings_pct=(100.0 * savings / baseline_spend) if baseline_spend > 0 else 0.0,
|
||||
)
|
||||
|
||||
|
||||
_SESSION_SQL = """
|
||||
SELECT
|
||||
session_id,
|
||||
COUNT(*)::bigint AS turns,
|
||||
EXTRACT(EPOCH FROM (MAX("endTime") - MIN("startTime"))) AS session_length_seconds,
|
||||
COALESCE(SUM(prompt_tokens), 0)::bigint AS prompt_tokens,
|
||||
COALESCE(SUM(completion_tokens), 0)::bigint AS completion_tokens,
|
||||
COALESCE(SUM(total_tokens), 0)::bigint AS total_tokens,
|
||||
COALESCE(SUM(spend), 0.0) AS actual_spend
|
||||
FROM "LiteLLM_SpendLogs"
|
||||
WHERE model_group = $1
|
||||
AND session_id IS NOT NULL
|
||||
AND "startTime" >= ($2::timestamptz AT TIME ZONE 'UTC')
|
||||
AND "startTime" < (($3::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC')
|
||||
GROUP BY session_id
|
||||
"""
|
||||
|
||||
_ROUTED_MODEL_SQL = """
|
||||
SELECT
|
||||
model,
|
||||
COALESCE(SUM(prompt_tokens), 0)::bigint AS prompt_tokens,
|
||||
COALESCE(SUM(completion_tokens), 0)::bigint AS completion_tokens
|
||||
FROM "LiteLLM_SpendLogs"
|
||||
WHERE model_group = $1
|
||||
AND session_id IS NOT NULL
|
||||
AND model IS NOT NULL
|
||||
AND "startTime" >= ($2::timestamptz AT TIME ZONE 'UTC')
|
||||
AND "startTime" < (($3::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC')
|
||||
GROUP BY model
|
||||
"""
|
||||
|
||||
|
||||
class _RawSessionRow(BaseModel):
|
||||
session_id: str
|
||||
turns: int
|
||||
session_length_seconds: float | None
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
total_tokens: int
|
||||
actual_spend: float
|
||||
|
||||
|
||||
class _RawRoutedModelRow(BaseModel):
|
||||
model: str
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
|
||||
|
||||
_SESSION_ROWS = TypeAdapter(tuple[_RawSessionRow, ...])
|
||||
_ROUTED_MODEL_ROWS = TypeAdapter(tuple[_RawRoutedModelRow, ...])
|
||||
|
||||
|
||||
async def compute_benchmarks(
|
||||
prisma_client: "PrismaClient",
|
||||
groups: tuple[tuple[str, str, str | None], ...],
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
) -> AutoRouterBenchmarksResponse:
|
||||
"""Run the windowed session + routed-model queries per group and fold them.
|
||||
|
||||
One pair of aggregate queries per auto-router group. Aggregation happens in
|
||||
Postgres; only per-session and per-model summaries cross the wire, never raw
|
||||
request rows.
|
||||
"""
|
||||
_, _, served_start, served_end = _clamp_window(start_date, end_date)
|
||||
summarized = [
|
||||
summarize_group(
|
||||
model_group,
|
||||
router_kind,
|
||||
await _fetch_sessions(prisma_client, model_group, served_start, served_end),
|
||||
await _fetch_routed_models(prisma_client, model_group, served_start, served_end),
|
||||
configured_baseline,
|
||||
)
|
||||
for model_group, router_kind, configured_baseline in groups
|
||||
]
|
||||
benchmarks = tuple(benchmark for benchmark in summarized if benchmark is not None)
|
||||
return AutoRouterBenchmarksResponse(start_date=served_start, end_date=served_end, groups=benchmarks)
|
||||
|
||||
|
||||
async def _fetch_sessions(
|
||||
prisma_client: "PrismaClient", model_group: str, start: str, end: str
|
||||
) -> tuple[_SessionRow, ...]:
|
||||
raw = await prisma_client.db.query_raw(_SESSION_SQL, model_group, start, end)
|
||||
return tuple(
|
||||
_SessionRow(
|
||||
session_id=row.session_id,
|
||||
turns=row.turns,
|
||||
session_length_seconds=row.session_length_seconds or 0.0,
|
||||
prompt_tokens=row.prompt_tokens,
|
||||
completion_tokens=row.completion_tokens,
|
||||
total_tokens=row.total_tokens,
|
||||
actual_spend=row.actual_spend,
|
||||
)
|
||||
for row in _SESSION_ROWS.validate_python(raw)
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_routed_models(
|
||||
prisma_client: "PrismaClient", model_group: str, start: str, end: str
|
||||
) -> tuple[_RoutedModelSpend, ...]:
|
||||
raw = await prisma_client.db.query_raw(_ROUTED_MODEL_SQL, model_group, start, end)
|
||||
return tuple(
|
||||
_RoutedModelSpend(model=row.model, prompt_tokens=row.prompt_tokens, completion_tokens=row.completion_tokens)
|
||||
for row in _ROUTED_MODEL_ROWS.validate_python(raw)
|
||||
)
|
||||
|
|
@ -284,6 +284,10 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
|
|||
quality_router_config: Optional[Dict] = None
|
||||
quality_router_default_model: Optional[str] = None
|
||||
|
||||
# auto-router benchmarks: pin the baseline the /auto_router/benchmarks endpoint
|
||||
# compares the routed model mix against; unset means derive it from traffic
|
||||
benchmark_baseline_model: Optional[str] = None
|
||||
|
||||
# Batch/File API Params
|
||||
s3_bucket_name: Optional[str] = None
|
||||
s3_encryption_key_id: Optional[str] = None
|
||||
|
|
|
|||
|
|
@ -3320,6 +3320,7 @@ all_litellm_params = (
|
|||
"adaptive_router_default_model",
|
||||
"quality_router_config",
|
||||
"quality_router_default_model",
|
||||
"benchmark_baseline_model",
|
||||
]
|
||||
+ list(StandardCallbackDynamicParams.__annotations__.keys())
|
||||
+ list(CustomPricingLiteLLMParams.model_fields.keys())
|
||||
|
|
|
|||
|
|
@ -0,0 +1,234 @@
|
|||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.abspath("../../../.."))
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.proxy.spend_tracking.auto_router_benchmarks import (
|
||||
BENCHMARKS_MAX_WINDOW_DAYS,
|
||||
_RoutedModelSpend,
|
||||
_SessionRow,
|
||||
_baseline_rates,
|
||||
_clamp_window,
|
||||
_derive_baseline_model,
|
||||
_pricing_candidates,
|
||||
auto_router_groups,
|
||||
compute_benchmarks,
|
||||
summarize_group,
|
||||
)
|
||||
|
||||
|
||||
def _rates(model: str) -> tuple[float, float]:
|
||||
info = litellm.get_model_info(model=model)
|
||||
return float(info["input_cost_per_token"] or 0.0), float(info["output_cost_per_token"] or 0.0)
|
||||
|
||||
|
||||
def _session(session_id: str, turns: int, secs: float, p_tok: int, c_tok: int, spend: float) -> _SessionRow:
|
||||
return _SessionRow(
|
||||
session_id=session_id,
|
||||
turns=turns,
|
||||
session_length_seconds=secs,
|
||||
prompt_tokens=p_tok,
|
||||
completion_tokens=c_tok,
|
||||
total_tokens=p_tok + c_tok,
|
||||
actual_spend=spend,
|
||||
)
|
||||
|
||||
|
||||
def test_baseline_prices_provider_prefixed_model():
|
||||
# SpendLogs stores models provider-prefixed while the cost map keys them bare;
|
||||
# both must resolve to the same non-zero rates or every savings number is wrong.
|
||||
prefixed = _baseline_rates("anthropic/claude-opus-5")
|
||||
bare = _baseline_rates("claude-opus-5")
|
||||
assert prefixed.input_cost_per_token > 0
|
||||
assert prefixed.output_cost_per_token > 0
|
||||
assert prefixed.input_cost_per_token == bare.input_cost_per_token
|
||||
assert prefixed.output_cost_per_token == bare.output_cost_per_token
|
||||
|
||||
|
||||
def test_baseline_falls_open_to_zero_for_unknown_model():
|
||||
rates = _baseline_rates("provider/model-that-does-not-exist")
|
||||
assert rates.input_cost_per_token == 0.0
|
||||
assert rates.output_cost_per_token == 0.0
|
||||
|
||||
|
||||
def test_pricing_candidates_include_prefixed_and_stripped():
|
||||
assert _pricing_candidates("anthropic/claude-opus-5") == ("anthropic/claude-opus-5", "claude-opus-5")
|
||||
assert _pricing_candidates("claude-opus-5") == ("claude-opus-5",)
|
||||
|
||||
|
||||
def test_counterfactual_uses_baseline_rates_on_actual_tokens():
|
||||
in_cost, out_cost = _rates("claude-opus-5")
|
||||
sessions = (_session("s1", turns=3, secs=100.0, p_tok=1000, c_tok=400, spend=0.002),)
|
||||
routed = (_RoutedModelSpend("anthropic/claude-haiku-4-5", 1000, 400),)
|
||||
bench = summarize_group("auto", "complexity", sessions, routed, configured_baseline="claude-opus-5")
|
||||
assert bench is not None
|
||||
expected_baseline_spend = 1000 * in_cost + 400 * out_cost
|
||||
assert bench.baseline_spend == pytest.approx(expected_baseline_spend)
|
||||
assert bench.savings == pytest.approx(expected_baseline_spend - 0.002)
|
||||
assert bench.savings_pct == pytest.approx(100.0 * (expected_baseline_spend - 0.002) / expected_baseline_spend)
|
||||
|
||||
|
||||
def test_session_metrics_are_averaged_across_sessions():
|
||||
sessions = (
|
||||
_session("s1", turns=10, secs=600.0, p_tok=800, c_tok=200, spend=0.001),
|
||||
_session("s2", turns=4, secs=200.0, p_tok=400, c_tok=100, spend=0.001),
|
||||
)
|
||||
routed = (_RoutedModelSpend("anthropic/claude-haiku-4-5", 1200, 300),)
|
||||
bench = summarize_group("auto", "complexity", sessions, routed, configured_baseline="claude-opus-5")
|
||||
assert bench is not None
|
||||
assert bench.sessions == 2
|
||||
assert bench.turns == 14
|
||||
assert bench.avg_turns_per_session == pytest.approx(7.0)
|
||||
assert bench.avg_session_length_seconds == pytest.approx(400.0)
|
||||
assert bench.total_tokens == 1500
|
||||
assert bench.avg_tokens_per_session == pytest.approx(750.0)
|
||||
|
||||
|
||||
def test_configured_baseline_overrides_derived():
|
||||
sessions = (_session("s1", turns=1, secs=10.0, p_tok=100, c_tok=100, spend=0.0001),)
|
||||
routed = (_RoutedModelSpend("anthropic/claude-haiku-4-5", 100, 100),)
|
||||
bench = summarize_group("auto", "complexity", sessions, routed, configured_baseline="claude-opus-5")
|
||||
assert bench is not None
|
||||
assert bench.baseline_model == "claude-opus-5"
|
||||
|
||||
|
||||
def test_derived_baseline_is_priciest_routed_model_not_highest_volume():
|
||||
# Haiku emits far more tokens but opus is the pricier model; the baseline must
|
||||
# be chosen on price, not token volume, or a cheap high-traffic model would be
|
||||
# mislabelled the flagship and collapse the savings figure.
|
||||
routed = (
|
||||
_RoutedModelSpend("anthropic/claude-haiku-4-5", 1_000_000, 1_000_000),
|
||||
_RoutedModelSpend("anthropic/claude-opus-5", 5, 5),
|
||||
)
|
||||
assert _derive_baseline_model(routed) == "anthropic/claude-opus-5"
|
||||
|
||||
|
||||
def test_empty_session_group_is_omitted():
|
||||
routed = (_RoutedModelSpend("anthropic/claude-opus-5", 10, 10),)
|
||||
assert summarize_group("auto", "complexity", (), routed, configured_baseline=None) is None
|
||||
|
||||
|
||||
def test_group_omitted_when_baseline_cannot_be_priced():
|
||||
# No configured baseline and no priceable routed model means there is no
|
||||
# honest counterfactual, so the group is dropped rather than reported at zero.
|
||||
sessions = (_session("s1", turns=1, secs=10.0, p_tok=100, c_tok=100, spend=0.0001),)
|
||||
routed = (_RoutedModelSpend("provider/unpriced-model", 100, 100),)
|
||||
assert summarize_group("auto", "complexity", sessions, routed, configured_baseline=None) is None
|
||||
|
||||
|
||||
def test_savings_pct_zero_when_baseline_unpriced_but_configured():
|
||||
# A configured baseline that the cost map cannot price yields a zero baseline
|
||||
# spend; savings_pct must not divide by zero.
|
||||
sessions = (_session("s1", turns=1, secs=10.0, p_tok=100, c_tok=100, spend=0.05),)
|
||||
routed = (_RoutedModelSpend("anthropic/claude-haiku-4-5", 100, 100),)
|
||||
bench = summarize_group("auto", "complexity", sessions, routed, configured_baseline="provider/unpriced")
|
||||
assert bench is not None
|
||||
assert bench.baseline_spend == 0.0
|
||||
assert bench.savings_pct == 0.0
|
||||
|
||||
|
||||
def test_window_clamped_to_max_days():
|
||||
_, _, served_start, served_end = _clamp_window("2026-01-01", "2026-07-29")
|
||||
assert served_end == "2026-07-29"
|
||||
# start is pulled forward to end - BENCHMARKS_MAX_WINDOW_DAYS
|
||||
assert served_start == "2026-06-29"
|
||||
assert (
|
||||
__import__("datetime").date.fromisoformat(served_end) - __import__("datetime").date.fromisoformat(served_start)
|
||||
).days == BENCHMARKS_MAX_WINDOW_DAYS
|
||||
|
||||
|
||||
def test_window_within_cap_is_preserved():
|
||||
start, _, served_start, served_end = _clamp_window("2026-07-20", "2026-07-29")
|
||||
assert served_start == "2026-07-20"
|
||||
assert served_end == "2026-07-29"
|
||||
|
||||
|
||||
class _FakeDB:
|
||||
def __init__(self, session_rows: list[dict[str, object]], model_rows: list[dict[str, object]]) -> None:
|
||||
self._session_rows = session_rows
|
||||
self._model_rows = model_rows
|
||||
self.queries: list[tuple[object, ...]] = []
|
||||
|
||||
async def query_raw(self, sql: str, *params: object) -> list[dict[str, object]]:
|
||||
self.queries.append(params)
|
||||
return self._session_rows if "GROUP BY session_id" in sql else self._model_rows
|
||||
|
||||
|
||||
class _FakePrisma:
|
||||
def __init__(self, db: _FakeDB) -> None:
|
||||
self.db = db
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compute_benchmarks_materializes_and_prices_rows_end_to_end():
|
||||
# Guards the two integration bugs the live rig caught: a tuple(await ...) that
|
||||
# produced an async generator instead of results, and untyped date params that
|
||||
# made Postgres compare timestamp >= text. This drives the real query ->
|
||||
# validate -> fold path with an injected fake DB returning dicts, the shape
|
||||
# prisma query_raw yields.
|
||||
session_rows = [
|
||||
{
|
||||
"session_id": "s1",
|
||||
"turns": 3,
|
||||
"session_length_seconds": 120.0,
|
||||
"prompt_tokens": 1000,
|
||||
"completion_tokens": 400,
|
||||
"total_tokens": 1400,
|
||||
"actual_spend": 0.002,
|
||||
},
|
||||
{
|
||||
"session_id": "s2",
|
||||
"turns": 1,
|
||||
"session_length_seconds": None,
|
||||
"prompt_tokens": 500,
|
||||
"completion_tokens": 100,
|
||||
"total_tokens": 600,
|
||||
"actual_spend": 0.001,
|
||||
},
|
||||
]
|
||||
model_rows = [{"model": "anthropic/claude-haiku-4-5", "prompt_tokens": 1500, "completion_tokens": 500}]
|
||||
db = _FakeDB(session_rows, model_rows)
|
||||
|
||||
response = await compute_benchmarks(
|
||||
_FakePrisma(db),
|
||||
groups=(("auto", "complexity", "claude-opus-5"),),
|
||||
start_date="2026-06-29",
|
||||
end_date="2026-07-29",
|
||||
)
|
||||
|
||||
assert len(response.groups) == 1
|
||||
bench = response.groups[0]
|
||||
assert bench.sessions == 2
|
||||
assert bench.turns == 4
|
||||
in_cost, out_cost = _rates("claude-opus-5")
|
||||
expected_baseline = (1000 + 500) * in_cost + (400 + 100) * out_cost
|
||||
assert bench.baseline_spend == pytest.approx(expected_baseline)
|
||||
assert bench.savings == pytest.approx(expected_baseline - 0.003)
|
||||
# a null session_length must fold as zero, not crash
|
||||
assert bench.avg_session_length_seconds == pytest.approx(60.0)
|
||||
# date params reach the DB as ISO strings so the ::timestamptz cast applies
|
||||
assert all(isinstance(p, str) for query in db.queries for p in query[1:])
|
||||
|
||||
|
||||
def test_auto_router_groups_enumerates_only_auto_routers_with_baseline_override():
|
||||
class _FakeRouter:
|
||||
model_list = [
|
||||
{
|
||||
"model_name": "auto",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/complexity_router",
|
||||
"benchmark_baseline_model": "claude-opus-5",
|
||||
},
|
||||
},
|
||||
{"model_name": "smart", "litellm_params": {"model": "auto_router/my-semantic-router"}},
|
||||
{"model_name": "claude-sonnet", "litellm_params": {"model": "anthropic/claude-sonnet-5"}},
|
||||
]
|
||||
|
||||
groups = auto_router_groups(_FakeRouter())
|
||||
assert ("auto", "complexity", "claude-opus-5") in groups
|
||||
assert ("smart", "semantic", None) in groups
|
||||
# a plain provider deployment is not an auto-router and must not appear
|
||||
assert all(model_group != "claude-sonnet" for model_group, _, _ in groups)
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
import { render, screen, waitFor } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { AutoRouterBenchmarksResponse, AutoRouterGroupBenchmark } from "@/components/networking";
|
||||
|
||||
const autoRouterBenchmarksCall = vi.fn();
|
||||
vi.mock("@/components/networking", () => ({
|
||||
autoRouterBenchmarksCall: (...args: unknown[]) => autoRouterBenchmarksCall(...args),
|
||||
}));
|
||||
|
||||
import AutoRouterBenchmarksPanel from "./AutoRouterBenchmarksPanel";
|
||||
|
||||
const group = (overrides: Partial<AutoRouterGroupBenchmark>): AutoRouterGroupBenchmark => ({
|
||||
model_group: "auto",
|
||||
router_kind: "complexity",
|
||||
baseline_model: "claude-opus-5",
|
||||
sessions: 75,
|
||||
turns: 1071,
|
||||
avg_turns_per_session: 14.3,
|
||||
avg_session_length_seconds: 2557.6,
|
||||
total_tokens: 142_000_000,
|
||||
avg_tokens_per_session: 1_899_868,
|
||||
actual_spend: 86.76,
|
||||
baseline_spend: 728.01,
|
||||
savings: 641.25,
|
||||
savings_pct: 88.1,
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const resolve = (groups: AutoRouterGroupBenchmark[]): void => {
|
||||
const response: AutoRouterBenchmarksResponse = { start_date: "2026-06-29", end_date: "2026-07-29", groups };
|
||||
autoRouterBenchmarksCall.mockResolvedValue(response);
|
||||
};
|
||||
|
||||
describe("AutoRouterBenchmarksPanel", () => {
|
||||
beforeEach(() => {
|
||||
autoRouterBenchmarksCall.mockReset();
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("renders the four session metrics with the routed group's numbers", async () => {
|
||||
resolve([group({})]);
|
||||
render(<AutoRouterBenchmarksPanel accessToken="sk-test" />);
|
||||
|
||||
await waitFor(() => expect(screen.getByText("Turns per session")).toBeInTheDocument());
|
||||
expect(screen.getByText("14.3")).toBeInTheDocument();
|
||||
// 2557.6s -> 42.6m
|
||||
expect(screen.getByText("42.6m")).toBeInTheDocument();
|
||||
// 1,899,868 tokens -> compact 1.9M
|
||||
expect(screen.getByText("1.9M")).toBeInTheDocument();
|
||||
// savings dollars + percent against the baseline model
|
||||
expect(screen.getByText("$641.25")).toBeInTheDocument();
|
||||
expect(screen.getByText(/88% vs claude-opus-5/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("labels the savings as an estimate and names the baseline it compares against", async () => {
|
||||
resolve([group({})]);
|
||||
render(<AutoRouterBenchmarksPanel accessToken="sk-test" />);
|
||||
|
||||
await waitFor(() => expect(screen.getByText("Estimated savings")).toBeInTheDocument());
|
||||
expect(screen.getByText(/does not model the caching a single-model baseline would have had/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders one card per auto-router group", async () => {
|
||||
resolve([group({ model_group: "auto" }), group({ model_group: "claude-auto", avg_turns_per_session: 4.1 })]);
|
||||
render(<AutoRouterBenchmarksPanel accessToken="sk-test" />);
|
||||
|
||||
await waitFor(() => expect(screen.getByText("auto")).toBeInTheDocument());
|
||||
expect(screen.getByText("claude-auto")).toBeInTheDocument();
|
||||
expect(screen.getByText("4.1")).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it("renders nothing when no auto-router has routed sessions", async () => {
|
||||
resolve([]);
|
||||
const { container } = render(<AutoRouterBenchmarksPanel accessToken="sk-test" />);
|
||||
|
||||
await waitFor(() => expect(autoRouterBenchmarksCall).toHaveBeenCalled());
|
||||
expect(container.textContent).not.toContain("Turns per session");
|
||||
});
|
||||
|
||||
it("does not call the endpoint without an access token", () => {
|
||||
render(<AutoRouterBenchmarksPanel accessToken={null} />);
|
||||
expect(autoRouterBenchmarksCall).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
"use client";
|
||||
|
||||
import React from "react";
|
||||
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { AutoRouterGroupBenchmark } from "@/components/networking";
|
||||
import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks";
|
||||
|
||||
const compactNumber = (n: number): string =>
|
||||
new Intl.NumberFormat("en-US", { notation: "compact", maximumFractionDigits: 1 }).format(n);
|
||||
|
||||
const currency = (n: number): string =>
|
||||
new Intl.NumberFormat("en-US", { style: "currency", currency: "USD", maximumFractionDigits: 2 }).format(n);
|
||||
|
||||
const durationLabel = (seconds: number): string => {
|
||||
if (seconds < 60) return `${Math.round(seconds)}s`;
|
||||
if (seconds < 3600) return `${(seconds / 60).toFixed(1)}m`;
|
||||
return `${(seconds / 3600).toFixed(1)}h`;
|
||||
};
|
||||
|
||||
const StatTile: React.FC<{ label: string; value: string; caption?: string }> = ({ label, value, caption }) => (
|
||||
<div className="rounded-lg border border-border bg-card p-4">
|
||||
<p className="text-sm text-muted-foreground">{label}</p>
|
||||
<p className="mt-1 text-2xl font-semibold text-foreground">{value}</p>
|
||||
{caption && <p className="mt-1 text-xs text-muted-foreground">{caption}</p>}
|
||||
</div>
|
||||
);
|
||||
|
||||
const GroupBenchmark: React.FC<{ group: AutoRouterGroupBenchmark }> = ({ group }) => (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<div className="flex flex-col gap-1 md:flex-row md:items-center md:justify-between">
|
||||
<CardTitle>{group.model_group}</CardTitle>
|
||||
<span className="text-xs text-muted-foreground">{group.sessions} sessions over the last 30 days</span>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent className="space-y-4">
|
||||
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2 lg:grid-cols-4">
|
||||
<StatTile label="Turns per session" value={group.avg_turns_per_session.toFixed(1)} />
|
||||
<StatTile label="Avg session length" value={durationLabel(group.avg_session_length_seconds)} />
|
||||
<StatTile label="Tokens per session" value={compactNumber(group.avg_tokens_per_session)} />
|
||||
<StatTile
|
||||
label="Estimated savings"
|
||||
value={currency(group.savings)}
|
||||
caption={`${group.savings_pct.toFixed(0)}% vs ${group.baseline_model}`}
|
||||
/>
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Savings estimate compares the routed model mix ({currency(group.actual_spend)}) against sending every request to{" "}
|
||||
{group.baseline_model} at list prices ({currency(group.baseline_spend)}); it does not model the caching a
|
||||
single-model baseline would have had
|
||||
</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
|
||||
interface AutoRouterBenchmarksPanelProps {
|
||||
accessToken: string | null;
|
||||
}
|
||||
|
||||
const AutoRouterBenchmarksPanel: React.FC<AutoRouterBenchmarksPanelProps> = ({ accessToken }) => {
|
||||
const { data, loading, error } = useAutoRouterBenchmarks(accessToken);
|
||||
|
||||
if (loading) {
|
||||
return <p className="py-8 text-center text-sm text-muted-foreground">Loading auto-router benchmarks...</p>;
|
||||
}
|
||||
if (error) {
|
||||
return null;
|
||||
}
|
||||
if (!data || data.groups.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<div>
|
||||
<h2 className="text-base font-semibold text-foreground">Benchmarks</h2>
|
||||
<p className="mt-1 text-sm text-muted-foreground">
|
||||
What each auto-router is buying you, measured over routed sessions in the last 30 days
|
||||
</p>
|
||||
</div>
|
||||
{data.groups.map((group) => (
|
||||
<GroupBenchmark key={group.model_group} group={group} />
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default AutoRouterBenchmarksPanel;
|
||||
|
|
@ -4,6 +4,7 @@ import React from "react";
|
|||
import { Form } from "antd";
|
||||
|
||||
import AddAutoRouterTab from "@/components/add_model/add_auto_router_tab";
|
||||
import AutoRouterBenchmarksPanel from "./AutoRouterBenchmarksPanel";
|
||||
|
||||
interface AutorouterTabProps {
|
||||
accessToken: string | null;
|
||||
|
|
@ -19,7 +20,8 @@ const AutorouterTab: React.FC<AutorouterTabProps> = ({ accessToken, userRole })
|
|||
}
|
||||
|
||||
return (
|
||||
<div className="w-full">
|
||||
<div className="w-full space-y-8">
|
||||
<AutoRouterBenchmarksPanel accessToken={accessToken} />
|
||||
<AddAutoRouterTab form={form} handleOk={() => form.resetFields()} accessToken={accessToken} userRole={userRole} />
|
||||
</div>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
import { useEffect, useMemo, useState } from "react";
|
||||
|
||||
import { autoRouterBenchmarksCall, AutoRouterBenchmarksResponse } from "@/components/networking";
|
||||
|
||||
const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000;
|
||||
|
||||
const toIsoDate = (d: Date): string => d.toISOString().slice(0, 10);
|
||||
|
||||
export interface AutoRouterBenchmarksState {
|
||||
data: AutoRouterBenchmarksResponse | null;
|
||||
loading: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
interface Resolved {
|
||||
key: string;
|
||||
data: AutoRouterBenchmarksResponse | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export const useAutoRouterBenchmarks = (accessToken: string | null): AutoRouterBenchmarksState => {
|
||||
const startDate = useMemo(() => toIsoDate(new Date(new Date().getTime() - THIRTY_DAYS_MS)), []);
|
||||
const endDate = useMemo(() => toIsoDate(new Date()), []);
|
||||
|
||||
const requestKey = accessToken ? `${accessToken}:${startDate}:${endDate}` : "";
|
||||
const [resolved, setResolved] = useState<Resolved | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!accessToken) {
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
autoRouterBenchmarksCall(accessToken, startDate, endDate)
|
||||
.then((data) => {
|
||||
if (!cancelled) setResolved({ key: requestKey, data, error: null });
|
||||
})
|
||||
.catch((e: unknown) => {
|
||||
if (!cancelled) {
|
||||
setResolved({ key: requestKey, data: null, error: e instanceof Error ? e.message : "Failed to load" });
|
||||
}
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [accessToken, startDate, endDate, requestKey]);
|
||||
|
||||
if (!accessToken) {
|
||||
return { data: null, loading: false, error: null };
|
||||
}
|
||||
if (resolved?.key !== requestKey) {
|
||||
return { data: null, loading: true, error: null };
|
||||
}
|
||||
return { data: resolved.data, loading: false, error: resolved.error };
|
||||
};
|
||||
|
|
@ -2007,6 +2007,39 @@ export const adminSpendLogsCall = async (accessToken: string) => {
|
|||
}
|
||||
};
|
||||
|
||||
export interface AutoRouterGroupBenchmark {
|
||||
model_group: string;
|
||||
router_kind: string;
|
||||
baseline_model: string;
|
||||
sessions: number;
|
||||
turns: number;
|
||||
avg_turns_per_session: number;
|
||||
avg_session_length_seconds: number;
|
||||
total_tokens: number;
|
||||
avg_tokens_per_session: number;
|
||||
actual_spend: number;
|
||||
baseline_spend: number;
|
||||
savings: number;
|
||||
savings_pct: number;
|
||||
}
|
||||
|
||||
export interface AutoRouterBenchmarksResponse {
|
||||
start_date: string;
|
||||
end_date: string;
|
||||
groups: AutoRouterGroupBenchmark[];
|
||||
}
|
||||
|
||||
export const autoRouterBenchmarksCall = async (
|
||||
accessToken: string,
|
||||
startDate: string,
|
||||
endDate: string,
|
||||
): Promise<AutoRouterBenchmarksResponse> => {
|
||||
return apiClient.get<AutoRouterBenchmarksResponse>(`/auto_router/benchmarks`, {
|
||||
accessToken,
|
||||
query: { start_date: startDate, end_date: endDate },
|
||||
});
|
||||
};
|
||||
|
||||
export const adminTopKeysCall = async (accessToken: string) => {
|
||||
try {
|
||||
let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/keys?limit=5` : `/global/spend/keys?limit=5`;
|
||||
|
|
|
|||
64
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
64
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -760,6 +760,34 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/auto_router/benchmarks": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* Get Auto Router Benchmarks
|
||||
* @description Session-level benchmarks for every configured auto-router.
|
||||
*
|
||||
* Admin-only. For each auto-router alias, reports turns per session, session
|
||||
* length, tokens per session, and the dollar savings of the routed model mix
|
||||
* versus sending every request to a single baseline model over the window.
|
||||
*
|
||||
* ``start_date`` / ``end_date`` are ``YYYY-MM-DD``; the window is clamped to
|
||||
* the most recent ``BENCHMARKS_MAX_WINDOW_DAYS`` days and the response echoes
|
||||
* the window actually served. Returns 404 when no auto-router is configured.
|
||||
*/
|
||||
get: operations["get_auto_router_benchmarks_auto_router_benchmarks_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/azure/{endpoint}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -25838,6 +25866,8 @@ export interface components {
|
|||
aws_secret_access_key?: string | null;
|
||||
/** Azure Ad Token */
|
||||
azure_ad_token?: string | null;
|
||||
/** Benchmark Baseline Model */
|
||||
benchmark_baseline_model?: string | null;
|
||||
/** Budget Duration */
|
||||
budget_duration?: string | null;
|
||||
/** Cache Creation Input Audio Token Cost */
|
||||
|
|
@ -33928,6 +33958,8 @@ export interface components {
|
|||
aws_secret_access_key?: string | null;
|
||||
/** Azure Ad Token */
|
||||
azure_ad_token?: string | null;
|
||||
/** Benchmark Baseline Model */
|
||||
benchmark_baseline_model?: string | null;
|
||||
/** Budget Duration */
|
||||
budget_duration?: string | null;
|
||||
/** Cache Creation Input Audio Token Cost */
|
||||
|
|
@ -35277,6 +35309,38 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
get_auto_router_benchmarks_auto_router_benchmarks_get: {
|
||||
parameters: {
|
||||
query: {
|
||||
start_date: string;
|
||||
end_date: string;
|
||||
};
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": unknown;
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
azure_proxy_route_azure__endpoint__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue