diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 18a927e7a44..55e2d233d7e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -19,6 +19,7 @@ from collections.abc import Mapping from datetime import datetime, timedelta, timezone from typing import ( TYPE_CHECKING, + Annotated, Any, AsyncGenerator, Callable, @@ -16322,6 +16323,52 @@ async def get_adaptive_router_state( return {"routers": snapshots} +@router.get( + "/auto_router/benchmarks", + tags=["auto_router"], + dependencies=[Depends(user_api_key_auth)], +) +async def get_auto_router_benchmarks( + start_date: str, + end_date: str, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + """Session-level benchmarks for every configured auto-router. + + Admin-only. For each auto-router alias, reports turns per session, session + length, tokens per session, and the dollar savings of the routed model mix + versus sending every request to a single baseline model over the window. + + ``start_date`` / ``end_date`` are ``YYYY-MM-DD``; the window is clamped to + the most recent ``BENCHMARKS_MAX_WINDOW_DAYS`` days and the response echoes + the window actually served. Returns 404 when no auto-router is configured. + """ + from litellm.proxy.spend_tracking.auto_router_benchmarks import auto_router_groups, compute_benchmarks + + if not _user_has_admin_view(user_api_key_dict): + raise HTTPException( + status_code=403, + detail={"error": CommonProxyErrors.not_allowed_access.value}, + ) + if llm_router is None: + raise HTTPException( + status_code=404, + detail={"error": "No auto_router is configured on this proxy."}, + ) + groups = auto_router_groups(llm_router) + if not groups: + raise HTTPException( + status_code=404, + detail={"error": "No auto_router is configured on this proxy."}, + ) + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + return await compute_benchmarks(prisma_client, groups, start_date, end_date) + + @router.get("/routes", dependencies=[Depends(user_api_key_auth)]) async def get_routes(): """ diff --git a/litellm/proxy/spend_tracking/auto_router_benchmarks.py b/litellm/proxy/spend_tracking/auto_router_benchmarks.py new file mode 100644 index 00000000000..e027f17c89c --- /dev/null +++ b/litellm/proxy/spend_tracking/auto_router_benchmarks.py @@ -0,0 +1,333 @@ +"""Session-level benchmarks for auto-router deployments. + +Answers the customer question "what is the auto-router actually buying me": how +many turns a routed session runs, how long it lasts, how many tokens it burns, +and how much cheaper the routed model mix is than sending every request to a +single baseline model. + +The four metrics are session-scoped, and ``session_id`` lives only on +``LiteLLM_SpendLogs`` (the per-request table), never on the daily rollups: a +session does not close on a day boundary, so it cannot be pre-aggregated the way +user/team spend is. This reads SpendLogs directly over a bounded window; the +durable answer is a per-session rollup table, tracked as a follow-up. + +Only rows whose ``model_group`` is a configured auto-router alias are counted. +That filter is load-bearing: the auto-router's own LLM-classifier sub-calls land +in the same session but carry the judge model's group, not the alias, so +grouping by the alias yields one row per routed turn with no classifier noise +and no separate ``call_type`` filter. +""" + +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING, Mapping, NamedTuple + +from pydantic import BaseModel, TypeAdapter + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model + +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient + from litellm.router import Router + + +BENCHMARKS_MAX_WINDOW_DAYS = 30 + + +class AutoRouterGroupBenchmark(BaseModel): + model_group: str + router_kind: str + baseline_model: str + sessions: int + turns: int + avg_turns_per_session: float + avg_session_length_seconds: float + total_tokens: int + avg_tokens_per_session: float + actual_spend: float + baseline_spend: float + savings: float + savings_pct: float + + +class AutoRouterBenchmarksResponse(BaseModel): + start_date: str + end_date: str + groups: tuple[AutoRouterGroupBenchmark, ...] + + +class _BaselineRates(NamedTuple): + model: str + input_cost_per_token: float + output_cost_per_token: float + + +class _SessionRow(NamedTuple): + session_id: str + turns: int + session_length_seconds: float + prompt_tokens: int + completion_tokens: int + total_tokens: int + actual_spend: float + + +def auto_router_groups(router: "Router") -> tuple[tuple[str, str, str | None], ...]: + """Enumerate ``(public model_group, router kind, configured baseline)`` per auto-router. + + ``model_group`` is the public alias clients send and SpendLogs records; the + ``litellm_params.model`` string is the ``auto_router/...`` discriminator. A + per-router ``benchmark_baseline_model`` override, when set, pins the baseline + to a fixed flagship; otherwise the caller derives it from the routed traffic. + """ + return tuple( + ( + str(entry["model_name"]), + kind, + _configured_baseline(entry.get("litellm_params")), + ) + for entry in (router.model_list or []) + if (model := _entry_model(entry)) is not None and (kind := classify_strategy_router_model(model)) is not None + ) + + +def _entry_model(entry: Mapping[str, object]) -> str | None: + params = entry.get("litellm_params") + if not isinstance(params, Mapping): + return None + model = params.get("model") + return model if isinstance(model, str) else None + + +def _configured_baseline(params: object) -> str | None: + if not isinstance(params, Mapping): + return None + baseline = params.get("benchmark_baseline_model") + return baseline if isinstance(baseline, str) and baseline else None + + +def _baseline_rates(model: str) -> _BaselineRates: + """Price a baseline model, trying provider-prefixed and bare name candidates. + + Daily/spend rows store models provider-prefixed (``anthropic/claude-opus-5``) + while the cost map often keys them bare, so a single lookup silently prices + zero. Falls open to zero rates for a fully unknown model, which surfaces as a + zero baseline (and zero savings) rather than a raised error. + """ + for candidate in _pricing_candidates(model): + try: + info = litellm.get_model_info(model=candidate) + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models + verbose_proxy_logger.debug("auto_router_benchmarks: no model info for %s (%s)", candidate, e) + continue + return _BaselineRates( + model=model, + input_cost_per_token=float(info.get("input_cost_per_token") or 0.0), + output_cost_per_token=float(info.get("output_cost_per_token") or 0.0), + ) + verbose_proxy_logger.warning( + "auto_router_benchmarks: baseline model %s is not priced; savings will read zero", model + ) + return _BaselineRates(model=model, input_cost_per_token=0.0, output_cost_per_token=0.0) + + +def _pricing_candidates(model: str) -> tuple[str, ...]: + stripped = model.split("/", 1)[1] if "/" in model else model + return tuple(dict.fromkeys((model, stripped))) + + +def _clamp_window(start_date: str, end_date: str) -> tuple[datetime, datetime, str, str]: + """Parse the range and enforce ``start >= end - BENCHMARKS_MAX_WINDOW_DAYS``. + + The scan is over the unbounded per-request table, so the window is capped the + way the tool-spend endpoint caps it; the returned start reflects the window + actually served. + """ + start = datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc) + end = datetime.fromisoformat(end_date).replace(tzinfo=timezone.utc) + floor = (end - timedelta(days=BENCHMARKS_MAX_WINDOW_DAYS)).replace(hour=0, minute=0, second=0, microsecond=0) + clamped_start = max(start, floor) + return clamped_start, end, clamped_start.date().isoformat(), end.date().isoformat() + + +def _derive_baseline_model(rows: tuple["_RoutedModelSpend", ...]) -> str | None: + """Pick the priciest model actually routed to in the window as the baseline. + + Uniform across all four router types and always a model the router really + used, so "vs baseline" is grounded in the deployment rather than a guess at + which tier is the flagship. Priced by blended per-token rate so a model is + not called the flagship purely because it emitted more tokens. + """ + priced = tuple((row.model, rate) for row in rows if (rate := _blended_rate(row.model)) is not None) + if not priced: + return None + return max(priced, key=lambda pair: pair[1])[0] + + +def _blended_rate(model: str) -> float | None: + for candidate in _pricing_candidates(model): + try: + info = litellm.get_model_info(model=candidate) + except Exception: # noqa: BLE001 # unmapped model, try next candidate + continue + return float(info.get("input_cost_per_token") or 0.0) + float(info.get("output_cost_per_token") or 0.0) + return None + + +class _RoutedModelSpend(NamedTuple): + model: str + prompt_tokens: int + completion_tokens: int + + +def summarize_group( + model_group: str, + router_kind: str, + session_rows: tuple[_SessionRow, ...], + routed_models: tuple[_RoutedModelSpend, ...], + configured_baseline: str | None, +) -> AutoRouterGroupBenchmark | None: + """Fold per-session and per-model rows into one group benchmark. + + Returns None when the window holds no routed sessions for the group, so an + idle auto-router is omitted rather than reported as a row of zeros. + """ + if not session_rows: + return None + baseline_model = configured_baseline or _derive_baseline_model(routed_models) + if baseline_model is None: + return None + rates = _baseline_rates(baseline_model) + + sessions = len(session_rows) + turns = sum(row.turns for row in session_rows) + total_tokens = sum(row.total_tokens for row in session_rows) + actual_spend = sum(row.actual_spend for row in session_rows) + baseline_spend = sum( + row.prompt_tokens * rates.input_cost_per_token + row.completion_tokens * rates.output_cost_per_token + for row in session_rows + ) + savings = baseline_spend - actual_spend + return AutoRouterGroupBenchmark( + model_group=model_group, + router_kind=router_kind, + baseline_model=baseline_model, + sessions=sessions, + turns=turns, + avg_turns_per_session=turns / sessions, + avg_session_length_seconds=sum(row.session_length_seconds for row in session_rows) / sessions, + total_tokens=total_tokens, + avg_tokens_per_session=total_tokens / sessions, + actual_spend=actual_spend, + baseline_spend=baseline_spend, + savings=savings, + savings_pct=(100.0 * savings / baseline_spend) if baseline_spend > 0 else 0.0, + ) + + +_SESSION_SQL = """ +SELECT + session_id, + COUNT(*)::bigint AS turns, + EXTRACT(EPOCH FROM (MAX("endTime") - MIN("startTime"))) AS session_length_seconds, + COALESCE(SUM(prompt_tokens), 0)::bigint AS prompt_tokens, + COALESCE(SUM(completion_tokens), 0)::bigint AS completion_tokens, + COALESCE(SUM(total_tokens), 0)::bigint AS total_tokens, + COALESCE(SUM(spend), 0.0) AS actual_spend +FROM "LiteLLM_SpendLogs" +WHERE model_group = $1 + AND session_id IS NOT NULL + AND "startTime" >= ($2::timestamptz AT TIME ZONE 'UTC') + AND "startTime" < (($3::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') +GROUP BY session_id +""" + +_ROUTED_MODEL_SQL = """ +SELECT + model, + COALESCE(SUM(prompt_tokens), 0)::bigint AS prompt_tokens, + COALESCE(SUM(completion_tokens), 0)::bigint AS completion_tokens +FROM "LiteLLM_SpendLogs" +WHERE model_group = $1 + AND session_id IS NOT NULL + AND model IS NOT NULL + AND "startTime" >= ($2::timestamptz AT TIME ZONE 'UTC') + AND "startTime" < (($3::timestamptz + INTERVAL '1 day') AT TIME ZONE 'UTC') +GROUP BY model +""" + + +class _RawSessionRow(BaseModel): + session_id: str + turns: int + session_length_seconds: float | None + prompt_tokens: int + completion_tokens: int + total_tokens: int + actual_spend: float + + +class _RawRoutedModelRow(BaseModel): + model: str + prompt_tokens: int + completion_tokens: int + + +_SESSION_ROWS = TypeAdapter(tuple[_RawSessionRow, ...]) +_ROUTED_MODEL_ROWS = TypeAdapter(tuple[_RawRoutedModelRow, ...]) + + +async def compute_benchmarks( + prisma_client: "PrismaClient", + groups: tuple[tuple[str, str, str | None], ...], + start_date: str, + end_date: str, +) -> AutoRouterBenchmarksResponse: + """Run the windowed session + routed-model queries per group and fold them. + + One pair of aggregate queries per auto-router group. Aggregation happens in + Postgres; only per-session and per-model summaries cross the wire, never raw + request rows. + """ + _, _, served_start, served_end = _clamp_window(start_date, end_date) + summarized = [ + summarize_group( + model_group, + router_kind, + await _fetch_sessions(prisma_client, model_group, served_start, served_end), + await _fetch_routed_models(prisma_client, model_group, served_start, served_end), + configured_baseline, + ) + for model_group, router_kind, configured_baseline in groups + ] + benchmarks = tuple(benchmark for benchmark in summarized if benchmark is not None) + return AutoRouterBenchmarksResponse(start_date=served_start, end_date=served_end, groups=benchmarks) + + +async def _fetch_sessions( + prisma_client: "PrismaClient", model_group: str, start: str, end: str +) -> tuple[_SessionRow, ...]: + raw = await prisma_client.db.query_raw(_SESSION_SQL, model_group, start, end) + return tuple( + _SessionRow( + session_id=row.session_id, + turns=row.turns, + session_length_seconds=row.session_length_seconds or 0.0, + prompt_tokens=row.prompt_tokens, + completion_tokens=row.completion_tokens, + total_tokens=row.total_tokens, + actual_spend=row.actual_spend, + ) + for row in _SESSION_ROWS.validate_python(raw) + ) + + +async def _fetch_routed_models( + prisma_client: "PrismaClient", model_group: str, start: str, end: str +) -> tuple[_RoutedModelSpend, ...]: + raw = await prisma_client.db.query_raw(_ROUTED_MODEL_SQL, model_group, start, end) + return tuple( + _RoutedModelSpend(model=row.model, prompt_tokens=row.prompt_tokens, completion_tokens=row.completion_tokens) + for row in _ROUTED_MODEL_ROWS.validate_python(raw) + ) diff --git a/litellm/types/router.py b/litellm/types/router.py index 28e4a8272e8..963c3cafe6c 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -284,6 +284,10 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): quality_router_config: Optional[Dict] = None quality_router_default_model: Optional[str] = None + # auto-router benchmarks: pin the baseline the /auto_router/benchmarks endpoint + # compares the routed model mix against; unset means derive it from traffic + benchmark_baseline_model: Optional[str] = None + # Batch/File API Params s3_bucket_name: Optional[str] = None s3_encryption_key_id: Optional[str] = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e4dfac48141..091ee5f898e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3320,6 +3320,7 @@ all_litellm_params = ( "adaptive_router_default_model", "quality_router_config", "quality_router_default_model", + "benchmark_baseline_model", ] + list(StandardCallbackDynamicParams.__annotations__.keys()) + list(CustomPricingLiteLLMParams.model_fields.keys()) diff --git a/tests/test_litellm/proxy/spend_tracking/test_auto_router_benchmarks.py b/tests/test_litellm/proxy/spend_tracking/test_auto_router_benchmarks.py new file mode 100644 index 00000000000..94aa9975491 --- /dev/null +++ b/tests/test_litellm/proxy/spend_tracking/test_auto_router_benchmarks.py @@ -0,0 +1,234 @@ +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../..")) + +import pytest + +import litellm +from litellm.proxy.spend_tracking.auto_router_benchmarks import ( + BENCHMARKS_MAX_WINDOW_DAYS, + _RoutedModelSpend, + _SessionRow, + _baseline_rates, + _clamp_window, + _derive_baseline_model, + _pricing_candidates, + auto_router_groups, + compute_benchmarks, + summarize_group, +) + + +def _rates(model: str) -> tuple[float, float]: + info = litellm.get_model_info(model=model) + return float(info["input_cost_per_token"] or 0.0), float(info["output_cost_per_token"] or 0.0) + + +def _session(session_id: str, turns: int, secs: float, p_tok: int, c_tok: int, spend: float) -> _SessionRow: + return _SessionRow( + session_id=session_id, + turns=turns, + session_length_seconds=secs, + prompt_tokens=p_tok, + completion_tokens=c_tok, + total_tokens=p_tok + c_tok, + actual_spend=spend, + ) + + +def test_baseline_prices_provider_prefixed_model(): + # SpendLogs stores models provider-prefixed while the cost map keys them bare; + # both must resolve to the same non-zero rates or every savings number is wrong. + prefixed = _baseline_rates("anthropic/claude-opus-5") + bare = _baseline_rates("claude-opus-5") + assert prefixed.input_cost_per_token > 0 + assert prefixed.output_cost_per_token > 0 + assert prefixed.input_cost_per_token == bare.input_cost_per_token + assert prefixed.output_cost_per_token == bare.output_cost_per_token + + +def test_baseline_falls_open_to_zero_for_unknown_model(): + rates = _baseline_rates("provider/model-that-does-not-exist") + assert rates.input_cost_per_token == 0.0 + assert rates.output_cost_per_token == 0.0 + + +def test_pricing_candidates_include_prefixed_and_stripped(): + assert _pricing_candidates("anthropic/claude-opus-5") == ("anthropic/claude-opus-5", "claude-opus-5") + assert _pricing_candidates("claude-opus-5") == ("claude-opus-5",) + + +def test_counterfactual_uses_baseline_rates_on_actual_tokens(): + in_cost, out_cost = _rates("claude-opus-5") + sessions = (_session("s1", turns=3, secs=100.0, p_tok=1000, c_tok=400, spend=0.002),) + routed = (_RoutedModelSpend("anthropic/claude-haiku-4-5", 1000, 400),) + bench = summarize_group("auto", "complexity", sessions, routed, configured_baseline="claude-opus-5") + assert bench is not None + expected_baseline_spend = 1000 * in_cost + 400 * out_cost + assert bench.baseline_spend == pytest.approx(expected_baseline_spend) + assert bench.savings == pytest.approx(expected_baseline_spend - 0.002) + assert bench.savings_pct == pytest.approx(100.0 * (expected_baseline_spend - 0.002) / expected_baseline_spend) + + +def test_session_metrics_are_averaged_across_sessions(): + sessions = ( + _session("s1", turns=10, secs=600.0, p_tok=800, c_tok=200, spend=0.001), + _session("s2", turns=4, secs=200.0, p_tok=400, c_tok=100, spend=0.001), + ) + routed = (_RoutedModelSpend("anthropic/claude-haiku-4-5", 1200, 300),) + bench = summarize_group("auto", "complexity", sessions, routed, configured_baseline="claude-opus-5") + assert bench is not None + assert bench.sessions == 2 + assert bench.turns == 14 + assert bench.avg_turns_per_session == pytest.approx(7.0) + assert bench.avg_session_length_seconds == pytest.approx(400.0) + assert bench.total_tokens == 1500 + assert bench.avg_tokens_per_session == pytest.approx(750.0) + + +def test_configured_baseline_overrides_derived(): + sessions = (_session("s1", turns=1, secs=10.0, p_tok=100, c_tok=100, spend=0.0001),) + routed = (_RoutedModelSpend("anthropic/claude-haiku-4-5", 100, 100),) + bench = summarize_group("auto", "complexity", sessions, routed, configured_baseline="claude-opus-5") + assert bench is not None + assert bench.baseline_model == "claude-opus-5" + + +def test_derived_baseline_is_priciest_routed_model_not_highest_volume(): + # Haiku emits far more tokens but opus is the pricier model; the baseline must + # be chosen on price, not token volume, or a cheap high-traffic model would be + # mislabelled the flagship and collapse the savings figure. + routed = ( + _RoutedModelSpend("anthropic/claude-haiku-4-5", 1_000_000, 1_000_000), + _RoutedModelSpend("anthropic/claude-opus-5", 5, 5), + ) + assert _derive_baseline_model(routed) == "anthropic/claude-opus-5" + + +def test_empty_session_group_is_omitted(): + routed = (_RoutedModelSpend("anthropic/claude-opus-5", 10, 10),) + assert summarize_group("auto", "complexity", (), routed, configured_baseline=None) is None + + +def test_group_omitted_when_baseline_cannot_be_priced(): + # No configured baseline and no priceable routed model means there is no + # honest counterfactual, so the group is dropped rather than reported at zero. + sessions = (_session("s1", turns=1, secs=10.0, p_tok=100, c_tok=100, spend=0.0001),) + routed = (_RoutedModelSpend("provider/unpriced-model", 100, 100),) + assert summarize_group("auto", "complexity", sessions, routed, configured_baseline=None) is None + + +def test_savings_pct_zero_when_baseline_unpriced_but_configured(): + # A configured baseline that the cost map cannot price yields a zero baseline + # spend; savings_pct must not divide by zero. + sessions = (_session("s1", turns=1, secs=10.0, p_tok=100, c_tok=100, spend=0.05),) + routed = (_RoutedModelSpend("anthropic/claude-haiku-4-5", 100, 100),) + bench = summarize_group("auto", "complexity", sessions, routed, configured_baseline="provider/unpriced") + assert bench is not None + assert bench.baseline_spend == 0.0 + assert bench.savings_pct == 0.0 + + +def test_window_clamped_to_max_days(): + _, _, served_start, served_end = _clamp_window("2026-01-01", "2026-07-29") + assert served_end == "2026-07-29" + # start is pulled forward to end - BENCHMARKS_MAX_WINDOW_DAYS + assert served_start == "2026-06-29" + assert ( + __import__("datetime").date.fromisoformat(served_end) - __import__("datetime").date.fromisoformat(served_start) + ).days == BENCHMARKS_MAX_WINDOW_DAYS + + +def test_window_within_cap_is_preserved(): + start, _, served_start, served_end = _clamp_window("2026-07-20", "2026-07-29") + assert served_start == "2026-07-20" + assert served_end == "2026-07-29" + + +class _FakeDB: + def __init__(self, session_rows: list[dict[str, object]], model_rows: list[dict[str, object]]) -> None: + self._session_rows = session_rows + self._model_rows = model_rows + self.queries: list[tuple[object, ...]] = [] + + async def query_raw(self, sql: str, *params: object) -> list[dict[str, object]]: + self.queries.append(params) + return self._session_rows if "GROUP BY session_id" in sql else self._model_rows + + +class _FakePrisma: + def __init__(self, db: _FakeDB) -> None: + self.db = db + + +@pytest.mark.asyncio +async def test_compute_benchmarks_materializes_and_prices_rows_end_to_end(): + # Guards the two integration bugs the live rig caught: a tuple(await ...) that + # produced an async generator instead of results, and untyped date params that + # made Postgres compare timestamp >= text. This drives the real query -> + # validate -> fold path with an injected fake DB returning dicts, the shape + # prisma query_raw yields. + session_rows = [ + { + "session_id": "s1", + "turns": 3, + "session_length_seconds": 120.0, + "prompt_tokens": 1000, + "completion_tokens": 400, + "total_tokens": 1400, + "actual_spend": 0.002, + }, + { + "session_id": "s2", + "turns": 1, + "session_length_seconds": None, + "prompt_tokens": 500, + "completion_tokens": 100, + "total_tokens": 600, + "actual_spend": 0.001, + }, + ] + model_rows = [{"model": "anthropic/claude-haiku-4-5", "prompt_tokens": 1500, "completion_tokens": 500}] + db = _FakeDB(session_rows, model_rows) + + response = await compute_benchmarks( + _FakePrisma(db), + groups=(("auto", "complexity", "claude-opus-5"),), + start_date="2026-06-29", + end_date="2026-07-29", + ) + + assert len(response.groups) == 1 + bench = response.groups[0] + assert bench.sessions == 2 + assert bench.turns == 4 + in_cost, out_cost = _rates("claude-opus-5") + expected_baseline = (1000 + 500) * in_cost + (400 + 100) * out_cost + assert bench.baseline_spend == pytest.approx(expected_baseline) + assert bench.savings == pytest.approx(expected_baseline - 0.003) + # a null session_length must fold as zero, not crash + assert bench.avg_session_length_seconds == pytest.approx(60.0) + # date params reach the DB as ISO strings so the ::timestamptz cast applies + assert all(isinstance(p, str) for query in db.queries for p in query[1:]) + + +def test_auto_router_groups_enumerates_only_auto_routers_with_baseline_override(): + class _FakeRouter: + model_list = [ + { + "model_name": "auto", + "litellm_params": { + "model": "auto_router/complexity_router", + "benchmark_baseline_model": "claude-opus-5", + }, + }, + {"model_name": "smart", "litellm_params": {"model": "auto_router/my-semantic-router"}}, + {"model_name": "claude-sonnet", "litellm_params": {"model": "anthropic/claude-sonnet-5"}}, + ] + + groups = auto_router_groups(_FakeRouter()) + assert ("auto", "complexity", "claude-opus-5") in groups + assert ("smart", "semantic", None) in groups + # a plain provider deployment is not an auto-router and must not appear + assert all(model_group != "claude-sonnet" for model_group, _, _ in groups) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksPanel.test.tsx new file mode 100644 index 00000000000..910c1acc927 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksPanel.test.tsx @@ -0,0 +1,87 @@ +import { render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { AutoRouterBenchmarksResponse, AutoRouterGroupBenchmark } from "@/components/networking"; + +const autoRouterBenchmarksCall = vi.fn(); +vi.mock("@/components/networking", () => ({ + autoRouterBenchmarksCall: (...args: unknown[]) => autoRouterBenchmarksCall(...args), +})); + +import AutoRouterBenchmarksPanel from "./AutoRouterBenchmarksPanel"; + +const group = (overrides: Partial): AutoRouterGroupBenchmark => ({ + model_group: "auto", + router_kind: "complexity", + baseline_model: "claude-opus-5", + sessions: 75, + turns: 1071, + avg_turns_per_session: 14.3, + avg_session_length_seconds: 2557.6, + total_tokens: 142_000_000, + avg_tokens_per_session: 1_899_868, + actual_spend: 86.76, + baseline_spend: 728.01, + savings: 641.25, + savings_pct: 88.1, + ...overrides, +}); + +const resolve = (groups: AutoRouterGroupBenchmark[]): void => { + const response: AutoRouterBenchmarksResponse = { start_date: "2026-06-29", end_date: "2026-07-29", groups }; + autoRouterBenchmarksCall.mockResolvedValue(response); +}; + +describe("AutoRouterBenchmarksPanel", () => { + beforeEach(() => { + autoRouterBenchmarksCall.mockReset(); + }); + afterEach(() => { + vi.clearAllMocks(); + }); + + it("renders the four session metrics with the routed group's numbers", async () => { + resolve([group({})]); + render(); + + await waitFor(() => expect(screen.getByText("Turns per session")).toBeInTheDocument()); + expect(screen.getByText("14.3")).toBeInTheDocument(); + // 2557.6s -> 42.6m + expect(screen.getByText("42.6m")).toBeInTheDocument(); + // 1,899,868 tokens -> compact 1.9M + expect(screen.getByText("1.9M")).toBeInTheDocument(); + // savings dollars + percent against the baseline model + expect(screen.getByText("$641.25")).toBeInTheDocument(); + expect(screen.getByText(/88% vs claude-opus-5/)).toBeInTheDocument(); + }); + + it("labels the savings as an estimate and names the baseline it compares against", async () => { + resolve([group({})]); + render(); + + await waitFor(() => expect(screen.getByText("Estimated savings")).toBeInTheDocument()); + expect(screen.getByText(/does not model the caching a single-model baseline would have had/)).toBeInTheDocument(); + }); + + it("renders one card per auto-router group", async () => { + resolve([group({ model_group: "auto" }), group({ model_group: "claude-auto", avg_turns_per_session: 4.1 })]); + render(); + + await waitFor(() => expect(screen.getByText("auto")).toBeInTheDocument()); + expect(screen.getByText("claude-auto")).toBeInTheDocument(); + expect(screen.getByText("4.1")).toBeInTheDocument(); + }); + + it("renders nothing when no auto-router has routed sessions", async () => { + resolve([]); + const { container } = render(); + + await waitFor(() => expect(autoRouterBenchmarksCall).toHaveBeenCalled()); + expect(container.textContent).not.toContain("Turns per session"); + }); + + it("does not call the endpoint without an access token", () => { + render(); + expect(autoRouterBenchmarksCall).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksPanel.tsx new file mode 100644 index 00000000000..0039a8367fe --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksPanel.tsx @@ -0,0 +1,89 @@ +"use client"; + +import React from "react"; + +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { AutoRouterGroupBenchmark } from "@/components/networking"; +import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks"; + +const compactNumber = (n: number): string => + new Intl.NumberFormat("en-US", { notation: "compact", maximumFractionDigits: 1 }).format(n); + +const currency = (n: number): string => + new Intl.NumberFormat("en-US", { style: "currency", currency: "USD", maximumFractionDigits: 2 }).format(n); + +const durationLabel = (seconds: number): string => { + if (seconds < 60) return `${Math.round(seconds)}s`; + if (seconds < 3600) return `${(seconds / 60).toFixed(1)}m`; + return `${(seconds / 3600).toFixed(1)}h`; +}; + +const StatTile: React.FC<{ label: string; value: string; caption?: string }> = ({ label, value, caption }) => ( +
+

{label}

+

{value}

+ {caption &&

{caption}

} +
+); + +const GroupBenchmark: React.FC<{ group: AutoRouterGroupBenchmark }> = ({ group }) => ( + + +
+ {group.model_group} + {group.sessions} sessions over the last 30 days +
+ + +
+ + + + +
+

+ Savings estimate compares the routed model mix ({currency(group.actual_spend)}) against sending every request to{" "} + {group.baseline_model} at list prices ({currency(group.baseline_spend)}); it does not model the caching a + single-model baseline would have had +

+ + +); + +interface AutoRouterBenchmarksPanelProps { + accessToken: string | null; +} + +const AutoRouterBenchmarksPanel: React.FC = ({ accessToken }) => { + const { data, loading, error } = useAutoRouterBenchmarks(accessToken); + + if (loading) { + return

Loading auto-router benchmarks...

; + } + if (error) { + return null; + } + if (!data || data.groups.length === 0) { + return null; + } + + return ( +
+
+

Benchmarks

+

+ What each auto-router is buying you, measured over routed sessions in the last 30 days +

+
+ {data.groups.map((group) => ( + + ))} +
+ ); +}; + +export default AutoRouterBenchmarksPanel; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx index 3474036528a..3d2962897d9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutorouterTab.tsx @@ -4,6 +4,7 @@ import React from "react"; import { Form } from "antd"; import AddAutoRouterTab from "@/components/add_model/add_auto_router_tab"; +import AutoRouterBenchmarksPanel from "./AutoRouterBenchmarksPanel"; interface AutorouterTabProps { accessToken: string | null; @@ -19,7 +20,8 @@ const AutorouterTab: React.FC = ({ accessToken, userRole }) } return ( -
+
+ form.resetFields()} accessToken={accessToken} userRole={userRole} />
); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts new file mode 100644 index 00000000000..74e662a896e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterBenchmarks.ts @@ -0,0 +1,54 @@ +import { useEffect, useMemo, useState } from "react"; + +import { autoRouterBenchmarksCall, AutoRouterBenchmarksResponse } from "@/components/networking"; + +const THIRTY_DAYS_MS = 30 * 24 * 60 * 60 * 1000; + +const toIsoDate = (d: Date): string => d.toISOString().slice(0, 10); + +export interface AutoRouterBenchmarksState { + data: AutoRouterBenchmarksResponse | null; + loading: boolean; + error: string | null; +} + +interface Resolved { + key: string; + data: AutoRouterBenchmarksResponse | null; + error: string | null; +} + +export const useAutoRouterBenchmarks = (accessToken: string | null): AutoRouterBenchmarksState => { + const startDate = useMemo(() => toIsoDate(new Date(new Date().getTime() - THIRTY_DAYS_MS)), []); + const endDate = useMemo(() => toIsoDate(new Date()), []); + + const requestKey = accessToken ? `${accessToken}:${startDate}:${endDate}` : ""; + const [resolved, setResolved] = useState(null); + + useEffect(() => { + if (!accessToken) { + return; + } + let cancelled = false; + autoRouterBenchmarksCall(accessToken, startDate, endDate) + .then((data) => { + if (!cancelled) setResolved({ key: requestKey, data, error: null }); + }) + .catch((e: unknown) => { + if (!cancelled) { + setResolved({ key: requestKey, data: null, error: e instanceof Error ? e.message : "Failed to load" }); + } + }); + return () => { + cancelled = true; + }; + }, [accessToken, startDate, endDate, requestKey]); + + if (!accessToken) { + return { data: null, loading: false, error: null }; + } + if (resolved?.key !== requestKey) { + return { data: null, loading: true, error: null }; + } + return { data: resolved.data, loading: false, error: resolved.error }; +}; diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 1331018c259..4865bda1f71 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -2007,6 +2007,39 @@ export const adminSpendLogsCall = async (accessToken: string) => { } }; +export interface AutoRouterGroupBenchmark { + model_group: string; + router_kind: string; + baseline_model: string; + sessions: number; + turns: number; + avg_turns_per_session: number; + avg_session_length_seconds: number; + total_tokens: number; + avg_tokens_per_session: number; + actual_spend: number; + baseline_spend: number; + savings: number; + savings_pct: number; +} + +export interface AutoRouterBenchmarksResponse { + start_date: string; + end_date: string; + groups: AutoRouterGroupBenchmark[]; +} + +export const autoRouterBenchmarksCall = async ( + accessToken: string, + startDate: string, + endDate: string, +): Promise => { + return apiClient.get(`/auto_router/benchmarks`, { + accessToken, + query: { start_date: startDate, end_date: endDate }, + }); +}; + export const adminTopKeysCall = async (accessToken: string) => { try { let url = proxyBaseUrl ? `${proxyBaseUrl}/global/spend/keys?limit=5` : `/global/spend/keys?limit=5`; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ed975c6be0a..7d9775f1d07 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -760,6 +760,34 @@ export interface paths { patch?: never; trace?: never; }; + "/auto_router/benchmarks": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Auto Router Benchmarks + * @description Session-level benchmarks for every configured auto-router. + * + * Admin-only. For each auto-router alias, reports turns per session, session + * length, tokens per session, and the dollar savings of the routed model mix + * versus sending every request to a single baseline model over the window. + * + * ``start_date`` / ``end_date`` are ``YYYY-MM-DD``; the window is clamped to + * the most recent ``BENCHMARKS_MAX_WINDOW_DAYS`` days and the response echoes + * the window actually served. Returns 404 when no auto-router is configured. + */ + get: operations["get_auto_router_benchmarks_auto_router_benchmarks_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/azure/{endpoint}": { parameters: { query?: never; @@ -25838,6 +25866,8 @@ export interface components { aws_secret_access_key?: string | null; /** Azure Ad Token */ azure_ad_token?: string | null; + /** Benchmark Baseline Model */ + benchmark_baseline_model?: string | null; /** Budget Duration */ budget_duration?: string | null; /** Cache Creation Input Audio Token Cost */ @@ -33928,6 +33958,8 @@ export interface components { aws_secret_access_key?: string | null; /** Azure Ad Token */ azure_ad_token?: string | null; + /** Benchmark Baseline Model */ + benchmark_baseline_model?: string | null; /** Budget Duration */ budget_duration?: string | null; /** Cache Creation Input Audio Token Cost */ @@ -35277,6 +35309,38 @@ export interface operations { }; }; }; + get_auto_router_benchmarks_auto_router_benchmarks_get: { + parameters: { + query: { + start_date: string; + end_date: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; azure_proxy_route_azure__endpoint__get: { parameters: { query?: never;