From 2e6d196684f7f7d8a5a344566724c0dfb9fa5bcf Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:47:10 -0700 Subject: [PATCH] feat(auto-router): surface escalation and abandonment quality signals Cost tells an operator the auto-router is cheaper, not that it is good. This adds the two quality signals that are already recorded on every request, so no new instrumentation is needed: escalation: a turn moved to a costlier model than the turn before it, inside one session. High precision, low recall. Only upward moves count, since a move down to a cheaper model is a deliberate price or latency trade, not evidence the router picked wrong. abandonment: the caller hung up mid-stream. Low precision, high recall, so it catches the giving-up that escalation structurally misses. They render side by side rather than blended, because they fail in opposite directions and one number would hide which of them fired. Both are computed for auto-routed sessions and for the same keys' directly addressed sessions, with identical definitions over LiteLLM_SpendLogs, so the two are comparable. The baseline is restricted to keys that actually used the router, since keys carry different model lists and pooling all non-router traffic would compare unlike populations. It is withheld entirely when the cohort is too small or when its traffic is not carrying caller-supplied session IDs, with the reason shown in place of a misleading rate. Neither signal can come from LiteLLM_AutoRouterSession: that rollup folds a session to per-model last-touch facts and discards turn order, and escalation is a question about order. Also moves avg saved per session up under the headline cost figure. --- .../proxy/db/autorouter_quality_signals.py | 206 ++++++++++++++++++ .../auto_router_endpoints.py | 181 +++++++++++++++ .../auto_router_endpoints.py | 54 +++++ .../db/test_autorouter_quality_signals.py | 186 ++++++++++++++++ .../test_auto_router_endpoints.py | 183 +++++++++++++++- .../AutoRouterBenchmarksTab.test.tsx | 107 ++++++++- .../_components/AutoRouterBenchmarksTab.tsx | 107 ++++++++- .../_components/autoRouterQualitySignals.ts | 33 +++ .../useAutoRouterQualitySignals.ts | 11 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 140 ++++++++++++ 10 files changed, 1195 insertions(+), 13 deletions(-) create mode 100644 litellm/proxy/db/autorouter_quality_signals.py create mode 100644 tests/test_litellm/proxy/db/test_autorouter_quality_signals.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterQualitySignals.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterQualitySignals.ts diff --git a/litellm/proxy/db/autorouter_quality_signals.py b/litellm/proxy/db/autorouter_quality_signals.py new file mode 100644 index 00000000000..d5467d5f3c6 --- /dev/null +++ b/litellm/proxy/db/autorouter_quality_signals.py @@ -0,0 +1,206 @@ +"""Quality signals for the auto-router, read from per-request spend logs. + +Cost tells an operator the router is cheaper; it cannot tell them the router is good. These +two signals are the cheapest honest evidence available without new instrumentation, because +both are already recorded on every request: + +``escalation`` a turn moved to a costlier model than the turn before it, inside one session. + When a caller does this they are saying the model they were given could not + do the job. High precision, low recall: it only sees callers who *can* switch + and bother to, so an API-only integration can report zero while suffering. + +``abandonment`` the caller hung up before the stream finished. Low precision, high recall: + it catches the giving-up that escalation structurally misses, but a dropped + connection and "I read enough" look the same from here. + +They are reported side by side rather than blended, because the two fail in opposite +directions and one number would hide which of them fired. + +Both are measured for auto-routed sessions and for the operator's own directly-addressed +sessions, with the same definitions over the same table, so the two can be read against each +other. That comparison is not an experiment: the cohorts self-select, and a deployment that +pins its hardest prompts to one model and routes only the easy ones will flatter itself. It +is directional evidence, and the surface that renders it says so. + +Neither signal can come from LiteLLM_AutoRouterSession. That rollup folds a session down to +per-model last-touch facts and deliberately discards turn order, and escalation is a question +about order. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from itertools import groupby +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_proxy_logger + +if TYPE_CHECKING: + from litellm.router import Router + +# A cohort smaller than this is noise: one unlucky session moves the rate by whole points, +# and a comparison drawn from it would read as fact. +MIN_COHORT_SESSIONS: Final = 20 + +# Below this share of turns carrying a caller-supplied session id, "sessions" in the +# non-routed cohort are mostly one-request artefacts of the fallback uuid, and any +# within-session signal computed over them is measuring the fallback, not the traffic. +MIN_SESSION_ID_COVERAGE: Final = 0.8 + + +class Turn: + """One request, reduced to what the two signals need. + + ``escalation`` needs the model and the order; ``abandonment`` needs the disconnect flag. + Kept as a plain object rather than a row dict so the signal functions state their inputs. + """ + + __slots__ = ("session_id", "model", "started_at", "client_disconnected", "router_name", "has_client_session_id") + + def __init__( + self, + *, + session_id: str, + model: str, + started_at: float, + client_disconnected: bool, + router_name: str | None, + has_client_session_id: bool, + ) -> None: + self.session_id = session_id + self.model = model + self.started_at = started_at + self.client_disconnected = client_disconnected + self.router_name = router_name + self.has_client_session_id = has_client_session_id + + +class CohortSignals: + """What one population evidences, and how much of it there was to look at.""" + + __slots__ = ("sessions", "escalation_rate_pct", "abandonment_rate_pct") + + def __init__( + self, + *, + sessions: int, + escalation_rate_pct: float | None, + abandonment_rate_pct: float | None, + ) -> None: + self.sessions = sessions + self.escalation_rate_pct = escalation_rate_pct + self.abandonment_rate_pct = abandonment_rate_pct + + +def rank_models_by_cost(router: "Router", models: Iterable[str]) -> Mapping[str, int]: + """Order models cheapest-first, by what one fixed reference request would cost on each. + + Rank has to come from a request, not from a rate card: a model dearer per output token + can be cheaper per cached token, so picking one rate to sort on orders cache-heavy + traffic backwards. Costing a single reference request through the pricing engine leaves + cache rates and tiered tables to the engine that already knows them. This mirrors how the + savings baseline picks "most expensive" (litellm/router_strategy/savings_baseline.py). + + Models that cannot be priced are absent from the result rather than sorted to an end, + because an unknown rate is not a low one; callers must treat a missing model as unrankable + and decline to judge the move, instead of reading it as cheap. + """ + from litellm.router_strategy.savings_baseline import Baseline, _priced + + quotes: Final = tuple((model, _priced(router, Baseline(model))) for model in dict.fromkeys(models)) + for model, quote in quotes: + if quote is None: + verbose_proxy_logger.debug("quality signals: cannot price %s, leaving it unranked", model) + priced: Final = sorted((quote[0], model) for model, quote in quotes if quote is not None) + return MappingProxyType({model: rank for rank, (_, model) in enumerate(priced)}) + + +def session_escalated(turns: Sequence[Turn], ranks: Mapping[str, int]) -> bool: + """True when some turn ran on a costlier model than the turn before it. + + Only upward moves count. A move down to a cheaper model is a caller trading quality for + price or latency deliberately, which is not evidence the router was wrong, and counting it + as a miss would make every cost-conscious user look like a dissatisfied one. + + A pair is skipped when either side is unrankable: an unpriceable model gives no direction, + and guessing one would invent the signal this function exists to measure. + """ + ordered: Final = sorted(turns, key=lambda turn: turn.started_at) + pairs: Final = ( + (ranks.get(previous.model), ranks.get(current.model)) for previous, current in zip(ordered, ordered[1:]) + ) + return any(from_rank is not None and to_rank is not None and to_rank > from_rank for from_rank, to_rank in pairs) + + +def could_escalate(turns: Sequence[Turn], ranks: Mapping[str, int], reachable: Iterable[str]) -> bool: + """True when something strictly costlier than where the session *started* was available. + + Eligibility is judged from the first turn, not from the priciest model the session went on + to use. Judging it from the maximum excludes the very sessions that escalated -- once a + session has moved up to the ceiling, "could it have moved up?" answers itself backwards -- + which would drive the measured rate toward zero exactly as the real one rose. + + A session that opened on the most capable model it could reach had nowhere to go, so its + silence says nothing about quality. Counting it as a satisfied session would let a + deployment lower its escalation rate by restricting keys, which is the opposite of what + this number should reward. + """ + ordered: Final = sorted(turns, key=lambda turn: turn.started_at) + opening_rank: Final = next((rank for turn in ordered if (rank := ranks.get(turn.model)) is not None), None) + if opening_rank is None: + return False + return any(reachable_rank > opening_rank for model in reachable if (reachable_rank := ranks.get(model)) is not None) + + +def _group_by_session(turns: Iterable[Turn]) -> Mapping[str, tuple[Turn, ...]]: + ordered: Final = sorted(turns, key=lambda turn: turn.session_id) + return MappingProxyType( + {session_id: tuple(group) for session_id, group in groupby(ordered, key=lambda turn: turn.session_id)} + ) + + +def signals_for_cohort( + turns: Sequence[Turn], + ranks: Mapping[str, int], + reachable: Iterable[str], +) -> CohortSignals: + """Both rates over the sessions that could have escalated. + + Abandonment shares escalation's denominator on purpose. The two numbers sit next to each + other in the UI and get read as one population; computing them over different sets would + make that reading wrong in a way nothing on screen would reveal. + """ + reachable_models: Final = tuple(reachable) + eligible: Final = tuple( + session_turns + for session_turns in _group_by_session(turns).values() + if could_escalate(session_turns, ranks, reachable_models) + ) + if not eligible: + return CohortSignals(sessions=0, escalation_rate_pct=None, abandonment_rate_pct=None) + + escalated: Final = sum(1 for session_turns in eligible if session_escalated(session_turns, ranks)) + eligible_turns: Final = tuple(turn for session_turns in eligible for turn in session_turns) + abandoned: Final = sum(1 for turn in eligible_turns if turn.client_disconnected) + return CohortSignals( + sessions=len(eligible), + escalation_rate_pct=round(100.0 * escalated / len(eligible), 1), + abandonment_rate_pct=round(100.0 * abandoned / len(eligible_turns), 1) if eligible_turns else None, + ) + + +def baseline_unavailable_reason(turns: Sequence[Turn], cohort: CohortSignals) -> str | None: + """Why the non-routed cohort cannot be shown, or None when it can. + + Session-id coverage is checked before size because the two failures need different words: + a deployment that never sends session ids has plenty of rows and no sessions, and telling + it "not enough traffic" would send it looking for volume it already has. + """ + if turns: + with_client_id: Final = sum(1 for turn in turns if turn.has_client_session_id) + if with_client_id / len(turns) < MIN_SESSION_ID_COVERAGE: + return "no_session_ids" + if cohort.sessions < MIN_COHORT_SESSIONS: + return "insufficient_sessions" + return None diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 141094f4d4c..88e5ae1b36d 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -6,6 +6,7 @@ POST /auto_router/test_routing - Route one prompt through an unsaved complexity- from collections.abc import Sequence from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final from pydantic import BaseModel, TypeAdapter @@ -34,6 +35,9 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( AutoRouterBenchmarkTotals, AutoRouterCacheBucket, AutoRouterCacheStats, + AutoRouterQualityCohort, + AutoRouterQualitySignals, + AutoRouterQualitySignalsResponse, AutoRouterRoutingTestRequest, AutoRouterRoutingTestResponse, RequestComplexityRouterConfig, @@ -454,3 +458,180 @@ async def get_auto_router_benchmarks( totals=_benchmark_totals(_summed_agg_row(rows)), groups=groups, ) + + +class _QualityTurnRow(BaseModel): + """One request from the window, carrying only what the two signals read.""" + + session_id: str + model: str + started_at: float + router_name: str | None + client_disconnected: bool + session_turn_count: int + api_key: str + + +_QUALITY_TURN_ROWS: Final = TypeAdapter(list[_QualityTurnRow]) + +# Both cohorts come from this one scan, so they cannot drift apart: same window, same +# session grouping, same disconnect test. `router_name` is NULL for a directly-addressed +# request, which is what separates the two populations downstream. +# +# Abandonment reads error_information.error_code rather than `status`, because a client +# disconnect is still billed as a success -- partial streamed spend is deliberately charged +# on disconnect -- so `status` cannot see it. +# +# session_turn_count is how many rows in the window share this session_id. There is no +# persisted flag for "the caller supplied this id rather than the spend writer minting a +# fallback uuid" (`_get_session_id_for_spend_log`), but a minted fallback is always unique to +# its one request, so a session_id that repeats could only have come from the caller. Counting +# it this way needs nothing beyond columns every deployment already writes, prompt storage on +# or off, which a check against proxy_server_request (only persisted when prompt storage is on) +# would not have been. +_QUALITY_SIGNALS_SQL: Final = """ +SELECT + session_id, + model, + EXTRACT(EPOCH FROM "startTime")::float8 AS started_at, + (metadata #>> '{routing_decision,router_model_name}') AS router_name, + COALESCE(metadata #>> '{error_information,error_code}' = '499', false) AS client_disconnected, + COUNT(*) OVER (PARTITION BY session_id)::int AS session_turn_count, + api_key +FROM "LiteLLM_SpendLogs" +WHERE "startTime" >= $1::timestamp AND "startTime" < $2::timestamp + AND session_id IS NOT NULL + AND model <> '' +""" + + +def _quality_signals_for( + turns: Sequence["_QualityTurnRow"], + router_name: str | None, + llm_router: "Router | None", +) -> AutoRouterQualitySignals: + """Signals for one router (or all of them) against that router's own comparable traffic. + + The baseline is drawn from the api_keys that used this router, not from every key on the + deployment. A key that can only reach one model has no escalation to make and would drag + the comparison toward zero for reasons that have nothing to do with routing quality; + restricting to keys that actually used the router keeps both sides of the comparison + inside the same reachable-model world. + + Reachability is judged against every model the key exercised anywhere in the window, not + against the models this one cohort happened to use. Scoping it to the cohort makes a + cohort that never escalated define its own ceiling, so every one of its sessions looks + like it had nowhere to go and the rate reads as unmeasurable rather than as zero -- + silently deleting exactly the well-behaved traffic the comparison exists to show. + """ + from litellm.proxy.db.autorouter_quality_signals import ( + Turn, + baseline_unavailable_reason, + rank_models_by_cost, + signals_for_cohort, + ) + + routed_rows: Final = tuple( + row for row in turns if row.router_name is not None and (router_name is None or row.router_name == router_name) + ) + router_keys: Final = frozenset(row.api_key for row in routed_rows) + baseline_rows: Final = tuple(row for row in turns if row.router_name is None and row.api_key in router_keys) + + reachable: Final = tuple(frozenset(row.model for row in turns if row.api_key in router_keys)) + ranks: Final = rank_models_by_cost(llm_router, reachable) if llm_router is not None else MappingProxyType({}) + + def as_turns(rows: Sequence[_QualityTurnRow]) -> tuple[Turn, ...]: + return tuple( + Turn( + session_id=row.session_id, + model=row.model, + started_at=row.started_at, + client_disconnected=row.client_disconnected, + router_name=row.router_name, + # A fallback uuid (_get_session_id_for_spend_log) is unique to its one + # request; only a caller-supplied session id repeats across rows. + has_client_session_id=row.session_turn_count > 1, + ) + for row in rows + ) + + routed: Final = signals_for_cohort(as_turns(routed_rows), ranks, reachable) + baseline: Final = signals_for_cohort(as_turns(baseline_rows), ranks, reachable) + unavailable: Final = baseline_unavailable_reason(as_turns(baseline_rows), baseline) + return AutoRouterQualitySignals( + router_name=router_name, + routed=AutoRouterQualityCohort( + sessions=routed.sessions, + escalation_rate_pct=routed.escalation_rate_pct, + abandonment_rate_pct=routed.abandonment_rate_pct, + ), + baseline=None + if unavailable + else AutoRouterQualityCohort( + sessions=baseline.sessions, + escalation_rate_pct=baseline.escalation_rate_pct, + abandonment_rate_pct=baseline.abandonment_rate_pct, + ), + baseline_unavailable_reason=unavailable, + ) + + +@router.get( + "/auto_router/quality_signals", + tags=("auto router",), + response_model=AutoRouterQualitySignalsResponse, +) +async def get_auto_router_quality_signals( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_date: Annotated[ + str | None, Query(description="YYYY-MM-DD UTC, inclusive (defaults to 30 days before end_date)") + ] = None, + end_date: Annotated[str | None, Query(description="YYYY-MM-DD UTC, inclusive (defaults to today)")] = None, +) -> AutoRouterQualitySignalsResponse: + """ + Quality signals for the auto-router dashboard: how often callers escalated to a costlier + model mid-session, and how often they hung up mid-stream, for auto-routed traffic and for + the same keys' directly-addressed traffic. + + Reads LiteLLM_SpendLogs rather than the per-session rollup, because escalation is a + question about turn order and the rollup folds order away. + + The two cohorts self-select, so this is directional evidence and not an experiment: a + deployment that pins its hardest prompts to one model and routes only the easy ones will + show a flattering baseline. + """ + from litellm.proxy.proxy_server import llm_router, prisma_client + + if user_api_key_dict.user_role not in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ): + raise HTTPException( + status_code=403, + detail="Only proxy admin roles can view auto-router quality signals across the deployment", + ) + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + + end_day: Final = ( + _parse_benchmark_day(end_date) + if end_date + else datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None) + ) + start_day: Final = _parse_benchmark_day(start_date) if start_date else end_day - timedelta(days=30) + if end_day < start_day: + raise HTTPException(status_code=400, detail="end_date must not be earlier than start_date") + + raw_rows: Final = await prisma_client.db.query_raw( + _QUALITY_SIGNALS_SQL, + start_day.isoformat(), + (end_day + timedelta(days=1)).isoformat(), + ) + turns: Final = _QUALITY_TURN_ROWS.validate_python(raw_rows or ()) + router_names: Final = tuple(sorted(frozenset(row.router_name for row in turns if row.router_name is not None))) + return AutoRouterQualitySignalsResponse( + start_date=start_day.strftime("%Y-%m-%d"), + end_date=end_day.strftime("%Y-%m-%d"), + totals=_quality_signals_for(turns, None, llm_router), + groups=tuple(_quality_signals_for(turns, name, llm_router) for name in router_names), + ) diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 6c8fb96a729..056ff0cc5a2 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -130,3 +130,57 @@ class AutoRouterBenchmarksResponse(BaseModel): routers_in_scope: int totals: AutoRouterBenchmarkTotals groups: tuple[AutoRouterBenchmarkGroup, ...] + + +class AutoRouterQualityCohort(BaseModel): + """One population's quality signals, over sessions that could have escalated. + + Both the routed and the non-routed cohort are measured with the same definitions over + the same table, so the two are directly comparable. ``sessions`` is the denominator for + every rate here: sessions whose key had a strictly more capable model available than the + one the session ran on, since a session that could not escalate cannot evidence a miss. + """ + + sessions: int = Field(description="Sessions in this cohort that had somewhere to escalate to") + escalation_rate_pct: float | None = Field( + description="Share of sessions where a turn moved to a more expensive model than the turn before it" + ) + abandonment_rate_pct: float | None = Field( + description="Share of turns the client disconnected before the stream finished" + ) + + +class AutoRouterQualitySignals(BaseModel): + """Quality signals for one auto-router, against the operator's own comparable traffic. + + Not a controlled experiment: the two cohorts self-select, so a deployment that pins its + hardest prompts to a fixed model and routes only the easy ones will show a flattering + baseline. The comparison is directional evidence, and the UI says so. + """ + + router_name: str | None = Field( + default=None, description="The router these signals cover; None when they cover all routers" + ) + routed: AutoRouterQualityCohort = Field(description="Sessions that went through the auto-router") + baseline: AutoRouterQualityCohort | None = Field( + description="Comparable sessions on a directly-addressed model, or None when there are too few " + "to compare, or too few carry a client-supplied session id to group into real sessions" + ) + baseline_unavailable_reason: str | None = Field( + default=None, + description="Why baseline is None: 'insufficient_sessions' or 'no_session_ids'", + ) + + +class AutoRouterQualitySignalsResponse(BaseModel): + """Quality signals for the auto-router dashboard, computed from per-request spend logs. + + Separate from the benchmarks endpoint because this reads LiteLLM_SpendLogs rather than the + per-session rollup: escalation is an ordered, per-turn question, and the rollup deliberately + folds ordering away. + """ + + start_date: str = Field(description="Window start day, YYYY-MM-DD UTC, inclusive") + end_date: str = Field(description="Window end day, YYYY-MM-DD UTC, inclusive") + totals: AutoRouterQualitySignals = Field(description="Signals over every auto-router in the window") + groups: tuple[AutoRouterQualitySignals, ...] = Field(description="Signals per auto-router") diff --git a/tests/test_litellm/proxy/db/test_autorouter_quality_signals.py b/tests/test_litellm/proxy/db/test_autorouter_quality_signals.py new file mode 100644 index 00000000000..f391d056252 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_autorouter_quality_signals.py @@ -0,0 +1,186 @@ +""" +Unit tests for the auto-router quality signal computation (escalation, abandonment). + +These test the pure logic in litellm.proxy.db.autorouter_quality_signals against +plain Turn objects and a fixed rank table, so they exercise the actual escalation/ +abandonment/eligibility rules without touching Postgres or a real Router. +""" + +from litellm.proxy.db.autorouter_quality_signals import ( + MIN_COHORT_SESSIONS, + MIN_SESSION_ID_COVERAGE, + Turn, + baseline_unavailable_reason, + could_escalate, + session_escalated, + signals_for_cohort, +) + +# haiku < sonnet < opus, matching how rank_models_by_cost would order real deployments +RANKS = {"haiku": 0, "sonnet": 1, "opus": 2} + + +def _turn(session_id="s1", model="sonnet", started_at=0.0, client_disconnected=False, has_client_session_id=True): + return Turn( + session_id=session_id, + model=model, + started_at=started_at, + client_disconnected=client_disconnected, + router_name="auto-router", + has_client_session_id=has_client_session_id, + ) + + +class TestSessionEscalated: + def test_upward_move_is_escalation(self): + turns = [_turn(model="sonnet", started_at=1), _turn(model="opus", started_at=2)] + assert session_escalated(turns, RANKS) is True + + def test_downward_move_is_not_escalation(self): + # This is the exact case the plan explicitly excludes: sonnet -> haiku is a cost + # or latency choice, not a quality miss, and must not be counted. + turns = [_turn(model="sonnet", started_at=1), _turn(model="haiku", started_at=2)] + assert session_escalated(turns, RANKS) is False + + def test_same_model_every_turn_is_not_escalation(self): + turns = [_turn(model="sonnet", started_at=1), _turn(model="sonnet", started_at=2)] + assert session_escalated(turns, RANKS) is False + + def test_single_turn_session_cannot_escalate(self): + assert session_escalated([_turn(model="sonnet", started_at=1)], RANKS) is False + + def test_escalation_detected_regardless_of_input_order(self): + # Turns can arrive from the DB in any order; escalation must be judged on + # started_at, not on list position. + turns = [_turn(model="opus", started_at=2), _turn(model="sonnet", started_at=1)] + assert session_escalated(turns, RANKS) is True + + def test_unrankable_model_does_not_count_as_escalation(self): + turns = [_turn(model="sonnet", started_at=1), _turn(model="mystery-model", started_at=2)] + assert session_escalated(turns, RANKS) is False + + def test_late_escalation_is_still_detected(self): + turns = [ + _turn(model="sonnet", started_at=1), + _turn(model="sonnet", started_at=2), + _turn(model="sonnet", started_at=3), + _turn(model="opus", started_at=4), + ] + assert session_escalated(turns, RANKS) is True + + +class TestCouldEscalate: + def test_session_on_cheapest_model_with_pricier_model_reachable_could_escalate(self): + turns = [_turn(model="haiku")] + assert could_escalate(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) is True + + def test_session_already_on_most_expensive_reachable_model_could_not_escalate(self): + # This is the case the plan calls out: a session pinned to the priciest model it + # could reach had nowhere to go, so it must not silently count as "no miss". + turns = [_turn(model="opus")] + assert could_escalate(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) is False + + def test_reachable_set_without_anything_costlier_cannot_escalate(self): + turns = [_turn(model="sonnet")] + assert could_escalate(turns, RANKS, reachable=["haiku", "sonnet"]) is False + + def test_no_rankable_turns_cannot_escalate(self): + turns = [_turn(model="unknown-model")] + assert could_escalate(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) is False + + def test_eligibility_is_judged_from_the_opening_turn_not_the_ceiling_reached(self): + # A session that started on sonnet and already escalated to opus must still count + # as "could escalate" -- judging from the max model used would make every session + # that actually escalated look ineligible, driving the measured rate toward zero + # exactly as the true rate rises. + turns = [_turn(model="sonnet", started_at=1), _turn(model="opus", started_at=2)] + assert could_escalate(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) is True + + def test_eligibility_uses_first_turn_by_time_not_by_list_order(self): + turns = [_turn(model="opus", started_at=2), _turn(model="sonnet", started_at=1)] + assert could_escalate(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) is True + + +class TestSignalsForCohort: + def test_escalation_rate_counts_only_eligible_sessions(self): + # s1 escalates and could have; s2 sits on opus (the ceiling) so it is excluded from + # the denominator entirely, not counted as a non-escalating session. + turns = [ + _turn(session_id="s1", model="sonnet", started_at=1), + _turn(session_id="s1", model="opus", started_at=2), + _turn(session_id="s2", model="opus", started_at=1), + ] + result = signals_for_cohort(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) + assert result.sessions == 1 + assert result.escalation_rate_pct == 100.0 + + def test_abandonment_counted_over_eligible_turns_only(self): + turns = [ + _turn(session_id="s1", model="haiku", started_at=1, client_disconnected=True), + _turn(session_id="s1", model="haiku", started_at=2, client_disconnected=False), + # s2 is on the ceiling model and ineligible; its disconnect must not be counted. + _turn(session_id="s2", model="opus", started_at=1, client_disconnected=True), + ] + result = signals_for_cohort(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) + assert result.sessions == 1 + assert result.abandonment_rate_pct == 50.0 + + def test_no_eligible_sessions_returns_none_rates_not_zero(self): + # Every session already on the ceiling model: zero would misleadingly claim + # "no miss detected" when in fact nothing could be measured. + turns = [_turn(session_id="s1", model="opus")] + result = signals_for_cohort(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) + assert result.sessions == 0 + assert result.escalation_rate_pct is None + assert result.abandonment_rate_pct is None + + def test_downward_switch_never_inflates_escalation_rate(self): + # Opens on sonnet (eligible: opus is reachable and costlier), then moves down to + # haiku. The downward move must not be read as an escalation. + turns = [ + _turn(session_id="s1", model="sonnet", started_at=1), + _turn(session_id="s1", model="haiku", started_at=2), + ] + result = signals_for_cohort(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) + assert result.sessions == 1 + assert result.escalation_rate_pct == 0.0 + + +class TestBaselineUnavailableReason: + def test_low_session_id_coverage_reported_before_size(self): + turns = [_turn(has_client_session_id=False) for _ in range(100)] + from litellm.proxy.db.autorouter_quality_signals import CohortSignals + + cohort = CohortSignals(sessions=100, escalation_rate_pct=1.0, abandonment_rate_pct=1.0) + assert baseline_unavailable_reason(turns, cohort) == "no_session_ids" + + def test_high_session_id_coverage_with_too_few_sessions_reports_size(self): + from litellm.proxy.db.autorouter_quality_signals import CohortSignals + + turns = [_turn(has_client_session_id=True) for _ in range(10)] + cohort = CohortSignals(sessions=MIN_COHORT_SESSIONS - 1, escalation_rate_pct=1.0, abandonment_rate_pct=1.0) + assert baseline_unavailable_reason(turns, cohort) == "insufficient_sessions" + + def test_sufficient_coverage_and_size_reports_available(self): + from litellm.proxy.db.autorouter_quality_signals import CohortSignals + + turns = [_turn(has_client_session_id=True) for _ in range(10)] + cohort = CohortSignals(sessions=MIN_COHORT_SESSIONS, escalation_rate_pct=1.0, abandonment_rate_pct=1.0) + assert baseline_unavailable_reason(turns, cohort) is None + + def test_coverage_exactly_at_floor_is_available(self): + from litellm.proxy.db.autorouter_quality_signals import CohortSignals + + n = 100 + with_id = int(n * MIN_SESSION_ID_COVERAGE) + turns = [_turn(has_client_session_id=True) for _ in range(with_id)] + [ + _turn(has_client_session_id=False) for _ in range(n - with_id) + ] + cohort = CohortSignals(sessions=MIN_COHORT_SESSIONS, escalation_rate_pct=1.0, abandonment_rate_pct=1.0) + assert baseline_unavailable_reason(turns, cohort) is None + + def test_no_turns_at_all_only_checked_against_size_floor(self): + from litellm.proxy.db.autorouter_quality_signals import CohortSignals + + cohort = CohortSignals(sessions=0, escalation_rate_pct=None, abandonment_rate_pct=None) + assert baseline_unavailable_reason([], cohort) == "insufficient_sessions" diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 888db031515..9c22575f8a5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -317,9 +317,7 @@ class TestAutoRouterBenchmarks: from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals totals = _benchmark_totals(self.ROW) - bucket_hits = ( - totals.cache.same_model.hits + totals.cache.first_visit.hits + totals.cache.return_to_tier.hits - ) + bucket_hits = totals.cache.same_model.hits + totals.cache.first_visit.hits + totals.cache.return_to_tier.hits assert bucket_hits == 27 assert totals.cache.hit_rate_pct == pytest.approx(100.0 * 28 / 38, abs=0.1) @@ -427,3 +425,182 @@ class TestAutoRouterBenchmarks: assert response.routers_in_scope == 1 assert response.groups[0].router_name == "live-auto" assert response.groups[0].saved_pct == response.totals.saved_pct == 75.0 + + +class TestAutoRouterQualitySignals: + """The endpoint's cohort split and its refusal to report a baseline it cannot stand behind. + + Escalation/abandonment arithmetic itself is covered in + tests/test_litellm/proxy/db/test_autorouter_quality_signals.py; these tests cover the + wiring around it -- which rows become which cohort, and when the baseline is withheld. + """ + + CHEAP = "openai/gpt-4o-mini" + PRICEY = "openai/gpt-4o" + + @staticmethod + def _row( + session_id: str, + model: str, + started_at: float, + *, + router_name: str | None = "live-auto", + client_disconnected: bool = False, + session_turn_count: int = 2, + api_key: str = "key-1", + ) -> dict: + return { + "session_id": session_id, + "model": model, + "started_at": started_at, + "router_name": router_name, + "client_disconnected": client_disconnected, + "session_turn_count": session_turn_count, + "api_key": api_key, + } + + @staticmethod + def _pricing_router() -> Router: + return Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": name, "api_key": "fake-key"}} + for name in (TestAutoRouterQualitySignals.CHEAP, TestAutoRouterQualitySignals.PRICEY) + ] + ) + + async def _call(self, rows: list[dict], monkeypatch: pytest.MonkeyPatch, **kwargs: object): + from litellm.proxy import proxy_server + from litellm.proxy.management_endpoints.auto_router_endpoints import ( + get_auto_router_quality_signals, + ) + + class _DB: + async def query_raw(self, sql: str, *params: object): + return rows + + monkeypatch.setattr(proxy_server, "prisma_client", type("P", (), {"db": _DB()})()) + monkeypatch.setattr(proxy_server, "llm_router", self._pricing_router()) + return await get_auto_router_quality_signals( + user_api_key_dict=ADMIN, + start_date="2026-08-01", + end_date="2026-08-02", + **kwargs, + ) + + @pytest.mark.asyncio + async def test_non_admin_roles_cannot_read_quality_signals(self): + from litellm.proxy.management_endpoints.auto_router_endpoints import ( + get_auto_router_quality_signals, + ) + + with pytest.raises(HTTPException) as err: + await get_auto_router_quality_signals( + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-x"), + start_date="2026-08-01", + end_date="2026-08-02", + ) + assert err.value.status_code == 403 + + @pytest.mark.asyncio + async def test_a_reversed_window_is_rejected(self, monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + from litellm.proxy.management_endpoints.auto_router_endpoints import ( + get_auto_router_quality_signals, + ) + + monkeypatch.setattr(proxy_server, "prisma_client", object()) + with pytest.raises(HTTPException) as err: + await get_auto_router_quality_signals( + user_api_key_dict=ADMIN, start_date="2026-08-05", end_date="2026-08-01" + ) + assert err.value.status_code == 400 + + @pytest.mark.asyncio + async def test_routed_escalation_is_measured_from_routed_rows(self, monkeypatch: pytest.MonkeyPatch): + rows = [ + self._row("s1", self.CHEAP, 1.0), + self._row("s1", self.PRICEY, 2.0), + ] + response = await self._call(rows, monkeypatch) + assert response.totals.routed.sessions == 1 + assert response.totals.routed.escalation_rate_pct == 100.0 + + @pytest.mark.asyncio + async def test_rows_without_a_routing_decision_are_the_baseline_not_the_routed_cohort( + self, monkeypatch: pytest.MonkeyPatch + ): + # Same key, same models; only the presence of a router name separates the cohorts. + # The routed session escalates and the direct one does not, so a cohort mix-up + # cannot produce these two numbers. + rows = [ + self._row("routed-1", self.CHEAP, 1.0), + self._row("routed-1", self.PRICEY, 2.0), + ] + rows += [self._row(f"direct-{i}", self.CHEAP, 1.0, router_name=None) for i in range(30)] + rows += [self._row(f"direct-{i}", self.CHEAP, 2.0, router_name=None) for i in range(30)] + response = await self._call(rows, monkeypatch) + assert response.totals.routed.escalation_rate_pct == 100.0 + assert response.totals.baseline is not None + assert response.totals.baseline.sessions == 30 + assert response.totals.baseline.escalation_rate_pct == 0.0 + + @pytest.mark.asyncio + async def test_baseline_withheld_when_sessions_are_mostly_one_request_fallback_ids( + self, monkeypatch: pytest.MonkeyPatch + ): + # session_turn_count == 1 marks a session id the spend writer minted rather than one + # the caller supplied. A cohort made of those is not sessions, and reporting a rate + # over it would describe the fallback rather than the traffic. + rows = [self._row("routed-1", self.CHEAP, 1.0), self._row("routed-1", self.PRICEY, 2.0)] + rows += [self._row(f"direct-{i}", self.CHEAP, 1.0, router_name=None, session_turn_count=1) for i in range(50)] + response = await self._call(rows, monkeypatch) + assert response.totals.baseline is None + assert response.totals.baseline_unavailable_reason == "no_session_ids" + + @pytest.mark.asyncio + async def test_baseline_withheld_when_there_is_too_little_comparable_traffic(self, monkeypatch: pytest.MonkeyPatch): + rows = [self._row("routed-1", self.CHEAP, 1.0), self._row("routed-1", self.PRICEY, 2.0)] + rows += [ + self._row("direct-1", self.CHEAP, 1.0, router_name=None), + self._row("direct-1", self.CHEAP, 2.0, router_name=None), + ] + response = await self._call(rows, monkeypatch) + assert response.totals.baseline is None + assert response.totals.baseline_unavailable_reason == "insufficient_sessions" + + @pytest.mark.asyncio + async def test_baseline_only_draws_on_keys_that_used_the_router(self, monkeypatch: pytest.MonkeyPatch): + # A key that never touched the router says nothing about the router, and its traffic + # must not be folded into the comparison. + rows = [self._row("routed-1", self.CHEAP, 1.0), self._row("routed-1", self.PRICEY, 2.0)] + rows += [self._row(f"other-{i}", self.CHEAP, 1.0, router_name=None, api_key="unrelated-key") for i in range(50)] + response = await self._call(rows, monkeypatch) + assert response.totals.baseline is None + + @pytest.mark.asyncio + async def test_groups_are_reported_per_router(self, monkeypatch: pytest.MonkeyPatch): + rows = [ + self._row("a1", self.CHEAP, 1.0, router_name="router-a"), + self._row("a1", self.PRICEY, 2.0, router_name="router-a"), + self._row("b1", self.CHEAP, 1.0, router_name="router-b"), + self._row("b1", self.CHEAP, 2.0, router_name="router-b"), + ] + response = await self._call(rows, monkeypatch) + by_name = {group.router_name: group for group in response.groups} + assert set(by_name) == {"router-a", "router-b"} + assert by_name["router-a"].routed.escalation_rate_pct == 100.0 + assert by_name["router-b"].routed.escalation_rate_pct == 0.0 + + @pytest.mark.asyncio + async def test_abandonment_reads_the_disconnect_flag_not_the_status(self, monkeypatch: pytest.MonkeyPatch): + # s2 gives the key a pricier model in the window, so s1 is eligible; without it s1 + # sits at its own ceiling and is correctly excluded, measuring nothing. + rows = [ + self._row("s1", self.CHEAP, 1.0, client_disconnected=True), + self._row("s1", self.CHEAP, 2.0, client_disconnected=False), + self._row("s2", self.PRICEY, 1.0), + self._row("s2", self.PRICEY, 2.0), + ] + response = await self._call(rows, monkeypatch) + assert response.totals.routed.sessions == 1 + assert response.totals.routed.abandonment_rate_pct == 50.0 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx index a5767383307..1806b748fb4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx @@ -1,10 +1,11 @@ import { fireEvent, render, screen } from "@testing-library/react"; import React from "react"; -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { ApiError } from "@/lib/http/client"; vi.mock("./useAutoRouterBenchmarks", () => ({ useAutoRouterBenchmarks: vi.fn() })); +vi.mock("./useAutoRouterQualitySignals", () => ({ useAutoRouterQualitySignals: vi.fn() })); import AutoRouterBenchmarksTab from "./AutoRouterBenchmarksTab"; import type { @@ -12,9 +13,12 @@ import type { AutoRouterBenchmarksResponse, AutoRouterCacheStats, } from "./autoRouterBenchmarks"; +import type { AutoRouterQualitySignalsResponse } from "./autoRouterQualitySignals"; import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks"; +import { useAutoRouterQualitySignals } from "./useAutoRouterQualitySignals"; type HookResult = ReturnType; +type QualityHookResult = ReturnType; const mockHook = (result: { data?: AutoRouterBenchmarksResponse; isPending?: boolean; error?: Error }) => { vi.mocked(useAutoRouterBenchmarks).mockReturnValue({ @@ -24,6 +28,14 @@ const mockHook = (result: { data?: AutoRouterBenchmarksResponse; isPending?: boo } as unknown as HookResult); }; +const mockQualityHook = (data?: AutoRouterQualitySignalsResponse) => { + vi.mocked(useAutoRouterQualitySignals).mockReturnValue({ + data, + isPending: false, + error: null, + } as unknown as QualityHookResult); +}; + const cache = (overrides: Partial = {}): AutoRouterCacheStats => ({ coverage_pct: 99.6, hit_rate_pct: 93.3, @@ -73,7 +85,35 @@ const response = (groups: AutoRouterBenchmarkGroup[], shared: Totals = totals()) const renderTab = () => render(); +const cohort = ( + overrides: Partial = {}, +): AutoRouterQualitySignalsResponse["totals"]["routed"] => ({ + sessions: 120, + escalation_rate_pct: 6.9, + abandonment_rate_pct: 2.4, + ...overrides, +}); + +const qualityResponse = ( + overrides: Partial = {}, +): AutoRouterQualitySignalsResponse => ({ + start_date: "2026-07-06", + end_date: "2026-08-05", + totals: { + router_name: null, + routed: cohort(), + baseline: cohort({ sessions: 80, escalation_rate_pct: 3.1, abandonment_rate_pct: 2.1 }), + baseline_unavailable_reason: null, + ...overrides, + }, + groups: [], +}); + describe("AutoRouterBenchmarksTab", () => { + beforeEach(() => { + mockQualityHook(undefined); + }); + it("leads with total estimated savings, before the three session-shape metrics", () => { mockHook({ data: response([group(), group({ router_name: "gpt-auto" })]) }); renderTab(); @@ -264,4 +304,69 @@ describe("AutoRouterBenchmarksTab", () => { expect(screen.getByRole("tab", { name: "30d" })).toBeInTheDocument(); expect(screen.getByText("All auto-routers")).toBeInTheDocument(); }); + + describe("quality signals", () => { + it("renders escalation and abandonment against the non-router baseline", () => { + mockHook({ data: response([group()]) }); + mockQualityHook(qualityResponse()); + renderTab(); + + expect(screen.getByText("Escalation rate")).toBeInTheDocument(); + expect(screen.getByText("6.9%")).toBeInTheDocument(); + expect(screen.getByText("vs. 3.1% on your non-router traffic")).toBeInTheDocument(); + + expect(screen.getByText("Stream abandonment")).toBeInTheDocument(); + expect(screen.getByText("2.4%")).toBeInTheDocument(); + expect(screen.getByText("vs. 2.1% on your non-router traffic")).toBeInTheDocument(); + }); + + it("does not render the quality card when there is no quality data yet", () => { + mockHook({ data: response([group()]) }); + mockQualityHook(undefined); + renderTab(); + + expect(screen.queryByText("Escalation rate")).not.toBeInTheDocument(); + }); + + it("explains why the baseline is missing instead of showing a misleading rate", () => { + mockHook({ data: response([group()]) }); + mockQualityHook( + qualityResponse({ + baseline: null, + baseline_unavailable_reason: "no_session_ids", + }), + ); + renderTab(); + + expect( + screen.getAllByText("Non-router traffic isn't sending session IDs, so it can't be grouped into sessions to compare"), + ).toHaveLength(2); + }); + + it("explains an insufficient-sessions baseline distinctly from a missing-session-id one", () => { + mockHook({ data: response([group()]) }); + mockQualityHook( + qualityResponse({ + baseline: null, + baseline_unavailable_reason: "insufficient_sessions", + }), + ); + renderTab(); + + expect(screen.getAllByText("Not enough comparable non-router traffic in this window to compare")).toHaveLength(2); + }); + + it("flags escalation as worse than baseline visually distinctly from a healthy rate", () => { + mockHook({ data: response([group()]) }); + mockQualityHook( + qualityResponse({ + routed: cohort({ escalation_rate_pct: 9.0 }), + baseline: cohort({ escalation_rate_pct: 3.0 }), + }), + ); + renderTab(); + + expect(screen.getByText("9.0%")).toHaveClass("text-destructive"); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx index ff0f52940b2..547b44ed966 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx @@ -28,8 +28,16 @@ import { type BenchmarkWindow, type BucketRow, } from "./autoRouterBenchmarks"; +import { + BASELINE_UNAVAILABLE_COPY, + ratePctLabel, + signalsFor, + type AutoRouterQualitySignals, + type AutoRouterQualitySignalsResponse, +} from "./autoRouterQualitySignals"; import { usd } from "./costOptimizationUtils"; import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks"; +import { useAutoRouterQualitySignals } from "./useAutoRouterQualitySignals"; const Message: React.FC<{ children: React.ReactNode }> = ({ children }) => (

{children}

@@ -64,6 +72,10 @@ const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => { {Math.abs(stats.saved_pct).toFixed(0)}% +
+

Avg saved per session

+

{usd(stats.saved_per_session)}

+
@@ -90,18 +102,72 @@ const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => {

{stats.turns.toLocaleString()}

-
-
-
Avg saved per session
-
{usd(stats.saved_per_session)}
-
-
); }; +const QualityMetric: React.FC<{ + label: string; + hint: string; + routed: number | null | undefined; + baseline: number | null | undefined; + baselineUnavailableReason: string | null | undefined; +}> = ({ label, hint, routed, baseline, baselineUnavailableReason }) => { + const worseThanBaseline = routed != null && baseline != null && routed > baseline; + return ( +
+ + + + {label} + + } + /> + {hint} + + +

+ {ratePctLabel(routed)} +

+

+ {baseline == null + ? (baselineUnavailableReason && BASELINE_UNAVAILABLE_COPY[baselineUnavailableReason]) || + "No comparable non-router traffic" + : `vs. ${ratePctLabel(baseline)} on your non-router traffic`} +

+
+ ); +}; + +const QualityCard: React.FC<{ signals: AutoRouterQualitySignals }> = ({ signals }) => ( + +

Quality signals

+ + + +); + const StackedTurnBar: React.FC<{ buckets: BucketRow[] }> = ({ buckets }) => { const segments = buckets.filter((b) => b.turns > 0); return ( @@ -233,9 +299,10 @@ interface BenchmarksBodyProps { error: unknown; data: AutoRouterBenchmarksResponse | undefined; selectedKey: string; + qualityData: AutoRouterQualitySignalsResponse | undefined; } -const BenchmarksBody: React.FC = ({ isPending, error, data, selectedKey }) => { +const BenchmarksBody: React.FC = ({ isPending, error, data, selectedKey, qualityData }) => { if (isPending) return Loading auto-router usage...; if (error instanceof ApiError && error.status === 403) { return Auto-router usage is visible to proxy admin roles only; @@ -245,9 +312,24 @@ const BenchmarksBody: React.FC = ({ isPending, error, data, const view = viewFor(data, selectedKey); const stats = view.stats; + const selectedGroup = data.groups.find((g) => groupKey(g) === selectedKey); + const signals = qualityData + ? signalsFor(qualityData, selectedKey === ALL_ROUTERS ? null : (selectedGroup?.router_name ?? null)) + : undefined; return ( <> - +
+ + {signals ? : null} +
+ + {signals ? ( +

+ Quality signals compare this router against the same keys' directly-addressed traffic. The two + populations self-select, so this is directional evidence rather than a controlled experiment: routing only + your easier prompts will flatter the comparison. +

+ ) : null}
@@ -281,6 +363,7 @@ interface AutoRouterBenchmarksTabProps { const AutoRouterBenchmarksTab: React.FC = ({ accessToken }) => { const [range, setRange] = useState("30d"); const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, range); + const { data: qualityData } = useAutoRouterQualitySignals(accessToken, range); const [selectedKey, setSelectedKey] = useState(ALL_ROUTERS); const groups = data?.groups ?? []; @@ -319,7 +402,13 @@ const AutoRouterBenchmarksTab: React.FC = ({ acces
- + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterQualitySignals.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterQualitySignals.ts new file mode 100644 index 00000000000..c0d8b1fd1ba --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterQualitySignals.ts @@ -0,0 +1,33 @@ +import type { components } from "@/lib/http/schema"; + +import { ALL_ROUTERS } from "./autoRouterBenchmarks"; + +export type AutoRouterQualitySignalsResponse = components["schemas"]["AutoRouterQualitySignalsResponse"]; +export type AutoRouterQualitySignals = components["schemas"]["AutoRouterQualitySignals"]; +export type AutoRouterQualityCohort = components["schemas"]["AutoRouterQualityCohort"]; + +export const BASELINE_UNAVAILABLE_COPY: Record = { + no_session_ids: "Non-router traffic isn't sending session IDs, so it can't be grouped into sessions to compare", + insufficient_sessions: "Not enough comparable non-router traffic in this window to compare", +}; + +export const signalsFor = ( + data: AutoRouterQualitySignalsResponse, + selectedRouterName: string | null, +): AutoRouterQualitySignals => { + if (selectedRouterName === null || selectedRouterName === ALL_ROUTERS) return data.totals; + return data.groups.find((group) => group.router_name === selectedRouterName) ?? data.totals; +}; + +export const ratePctLabel = (value: number | null | undefined): string => + value === null || value === undefined ? "—" : `${value.toFixed(1)}%`; + +/** Positive when the router escalates more often than the operator's own direct traffic. */ +export const deltaVsBaseline = ( + routed: number | null | undefined, + baseline: number | null | undefined, +): number | null => { + if (routed === null || routed === undefined) return null; + if (baseline === null || baseline === undefined) return null; + return routed - baseline; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterQualitySignals.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterQualitySignals.ts new file mode 100644 index 00000000000..5e8a0cc445b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterQualitySignals.ts @@ -0,0 +1,11 @@ +import { $api } from "@/lib/http/api"; + +import { windowFor, type BenchmarkWindow } from "./autoRouterBenchmarks"; + +export const useAutoRouterQualitySignals = (accessToken: string | null, range: BenchmarkWindow) => + $api.useQuery( + "get", + "/auto_router/quality_signals", + { params: { query: windowFor(range, new Date()) } }, + { enabled: Boolean(accessToken), retry: false }, + ); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f1660e77ad9..0fcf206fb08 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -807,6 +807,35 @@ export interface paths { patch?: never; trace?: never; }; + "/auto_router/quality_signals": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Auto Router Quality Signals + * @description Quality signals for the auto-router dashboard: how often callers escalated to a costlier + * model mid-session, and how often they hung up mid-stream, for auto-routed traffic and for + * the same keys' directly-addressed traffic. + * + * Reads LiteLLM_SpendLogs rather than the per-session rollup, because escalation is a + * question about turn order and the rollup folds order away. + * + * The two cohorts self-select, so this is directional evidence and not an experiment: a + * deployment that pins its hardest prompts to one model and routes only the easy ones will + * show a flattering baseline. + */ + get: operations["get_auto_router_quality_signals_auto_router_quality_signals_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/auto_router/test_routing": { parameters: { query?: never; @@ -21539,6 +21568,83 @@ export interface components { /** System Prompt */ system_prompt: string; }; + /** + * AutoRouterQualityCohort + * @description One population's quality signals, over sessions that could have escalated. + * + * Both the routed and the non-routed cohort are measured with the same definitions over + * the same table, so the two are directly comparable. ``sessions`` is the denominator for + * every rate here: sessions whose key had a strictly more capable model available than the + * one the session ran on, since a session that could not escalate cannot evidence a miss. + */ + AutoRouterQualityCohort: { + /** + * Abandonment Rate Pct + * @description Share of turns the client disconnected before the stream finished + */ + abandonment_rate_pct: number | null; + /** + * Escalation Rate Pct + * @description Share of sessions where a turn moved to a more expensive model than the turn before it + */ + escalation_rate_pct: number | null; + /** + * Sessions + * @description Sessions in this cohort that had somewhere to escalate to + */ + sessions: number; + }; + /** + * AutoRouterQualitySignals + * @description Quality signals for one auto-router, against the operator's own comparable traffic. + * + * Not a controlled experiment: the two cohorts self-select, so a deployment that pins its + * hardest prompts to a fixed model and routes only the easy ones will show a flattering + * baseline. The comparison is directional evidence, and the UI says so. + */ + AutoRouterQualitySignals: { + /** @description Comparable sessions on a directly-addressed model, or None when there are too few to compare, or too few carry a client-supplied session id to group into real sessions */ + baseline: components["schemas"]["AutoRouterQualityCohort"] | null; + /** + * Baseline Unavailable Reason + * @description Why baseline is None: 'insufficient_sessions' or 'no_session_ids' + */ + baseline_unavailable_reason?: string | null; + /** @description Sessions that went through the auto-router */ + routed: components["schemas"]["AutoRouterQualityCohort"]; + /** + * Router Name + * @description The router these signals cover; None when they cover all routers + */ + router_name?: string | null; + }; + /** + * AutoRouterQualitySignalsResponse + * @description Quality signals for the auto-router dashboard, computed from per-request spend logs. + * + * Separate from the benchmarks endpoint because this reads LiteLLM_SpendLogs rather than the + * per-session rollup: escalation is an ordered, per-turn question, and the rollup deliberately + * folds ordering away. + */ + AutoRouterQualitySignalsResponse: { + /** + * End Date + * @description Window end day, YYYY-MM-DD UTC, inclusive + */ + end_date: string; + /** + * Groups + * @description Signals per auto-router + */ + groups: components["schemas"]["AutoRouterQualitySignals"][]; + /** + * Start Date + * @description Window start day, YYYY-MM-DD UTC, inclusive + */ + start_date: string; + /** @description Signals over every auto-router in the window */ + totals: components["schemas"]["AutoRouterQualitySignals"]; + }; /** * AutoRouterRoutingTestRequest * @description A single prompt to classify against a complexity-router config that need not be saved yet. @@ -36797,6 +36903,40 @@ export interface operations { }; }; }; + get_auto_router_quality_signals_auto_router_quality_signals_get: { + parameters: { + query?: { + /** @description YYYY-MM-DD UTC, inclusive (defaults to 30 days before end_date) */ + start_date?: string | null; + /** @description YYYY-MM-DD UTC, inclusive (defaults to today) */ + end_date?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AutoRouterQualitySignalsResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; preview_auto_router_routing_auto_router_test_routing_post: { parameters: { query?: never;