diff --git a/litellm/proxy/db/autorouter_quality_signals.py b/litellm/proxy/db/autorouter_quality_signals.py new file mode 100644 index 00000000000..d5467d5f3c6 --- /dev/null +++ b/litellm/proxy/db/autorouter_quality_signals.py @@ -0,0 +1,206 @@ +"""Quality signals for the auto-router, read from per-request spend logs. + +Cost tells an operator the router is cheaper; it cannot tell them the router is good. These +two signals are the cheapest honest evidence available without new instrumentation, because +both are already recorded on every request: + +``escalation`` a turn moved to a costlier model than the turn before it, inside one session. + When a caller does this they are saying the model they were given could not + do the job. High precision, low recall: it only sees callers who *can* switch + and bother to, so an API-only integration can report zero while suffering. + +``abandonment`` the caller hung up before the stream finished. Low precision, high recall: + it catches the giving-up that escalation structurally misses, but a dropped + connection and "I read enough" look the same from here. + +They are reported side by side rather than blended, because the two fail in opposite +directions and one number would hide which of them fired. + +Both are measured for auto-routed sessions and for the operator's own directly-addressed +sessions, with the same definitions over the same table, so the two can be read against each +other. That comparison is not an experiment: the cohorts self-select, and a deployment that +pins its hardest prompts to one model and routes only the easy ones will flatter itself. It +is directional evidence, and the surface that renders it says so. + +Neither signal can come from LiteLLM_AutoRouterSession. That rollup folds a session down to +per-model last-touch facts and deliberately discards turn order, and escalation is a question +about order. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping, Sequence +from itertools import groupby +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_proxy_logger + +if TYPE_CHECKING: + from litellm.router import Router + +# A cohort smaller than this is noise: one unlucky session moves the rate by whole points, +# and a comparison drawn from it would read as fact. +MIN_COHORT_SESSIONS: Final = 20 + +# Below this share of turns carrying a caller-supplied session id, "sessions" in the +# non-routed cohort are mostly one-request artefacts of the fallback uuid, and any +# within-session signal computed over them is measuring the fallback, not the traffic. +MIN_SESSION_ID_COVERAGE: Final = 0.8 + + +class Turn: + """One request, reduced to what the two signals need. + + ``escalation`` needs the model and the order; ``abandonment`` needs the disconnect flag. + Kept as a plain object rather than a row dict so the signal functions state their inputs. + """ + + __slots__ = ("session_id", "model", "started_at", "client_disconnected", "router_name", "has_client_session_id") + + def __init__( + self, + *, + session_id: str, + model: str, + started_at: float, + client_disconnected: bool, + router_name: str | None, + has_client_session_id: bool, + ) -> None: + self.session_id = session_id + self.model = model + self.started_at = started_at + self.client_disconnected = client_disconnected + self.router_name = router_name + self.has_client_session_id = has_client_session_id + + +class CohortSignals: + """What one population evidences, and how much of it there was to look at.""" + + __slots__ = ("sessions", "escalation_rate_pct", "abandonment_rate_pct") + + def __init__( + self, + *, + sessions: int, + escalation_rate_pct: float | None, + abandonment_rate_pct: float | None, + ) -> None: + self.sessions = sessions + self.escalation_rate_pct = escalation_rate_pct + self.abandonment_rate_pct = abandonment_rate_pct + + +def rank_models_by_cost(router: "Router", models: Iterable[str]) -> Mapping[str, int]: + """Order models cheapest-first, by what one fixed reference request would cost on each. + + Rank has to come from a request, not from a rate card: a model dearer per output token + can be cheaper per cached token, so picking one rate to sort on orders cache-heavy + traffic backwards. Costing a single reference request through the pricing engine leaves + cache rates and tiered tables to the engine that already knows them. This mirrors how the + savings baseline picks "most expensive" (litellm/router_strategy/savings_baseline.py). + + Models that cannot be priced are absent from the result rather than sorted to an end, + because an unknown rate is not a low one; callers must treat a missing model as unrankable + and decline to judge the move, instead of reading it as cheap. + """ + from litellm.router_strategy.savings_baseline import Baseline, _priced + + quotes: Final = tuple((model, _priced(router, Baseline(model))) for model in dict.fromkeys(models)) + for model, quote in quotes: + if quote is None: + verbose_proxy_logger.debug("quality signals: cannot price %s, leaving it unranked", model) + priced: Final = sorted((quote[0], model) for model, quote in quotes if quote is not None) + return MappingProxyType({model: rank for rank, (_, model) in enumerate(priced)}) + + +def session_escalated(turns: Sequence[Turn], ranks: Mapping[str, int]) -> bool: + """True when some turn ran on a costlier model than the turn before it. + + Only upward moves count. A move down to a cheaper model is a caller trading quality for + price or latency deliberately, which is not evidence the router was wrong, and counting it + as a miss would make every cost-conscious user look like a dissatisfied one. + + A pair is skipped when either side is unrankable: an unpriceable model gives no direction, + and guessing one would invent the signal this function exists to measure. + """ + ordered: Final = sorted(turns, key=lambda turn: turn.started_at) + pairs: Final = ( + (ranks.get(previous.model), ranks.get(current.model)) for previous, current in zip(ordered, ordered[1:]) + ) + return any(from_rank is not None and to_rank is not None and to_rank > from_rank for from_rank, to_rank in pairs) + + +def could_escalate(turns: Sequence[Turn], ranks: Mapping[str, int], reachable: Iterable[str]) -> bool: + """True when something strictly costlier than where the session *started* was available. + + Eligibility is judged from the first turn, not from the priciest model the session went on + to use. Judging it from the maximum excludes the very sessions that escalated -- once a + session has moved up to the ceiling, "could it have moved up?" answers itself backwards -- + which would drive the measured rate toward zero exactly as the real one rose. + + A session that opened on the most capable model it could reach had nowhere to go, so its + silence says nothing about quality. Counting it as a satisfied session would let a + deployment lower its escalation rate by restricting keys, which is the opposite of what + this number should reward. + """ + ordered: Final = sorted(turns, key=lambda turn: turn.started_at) + opening_rank: Final = next((rank for turn in ordered if (rank := ranks.get(turn.model)) is not None), None) + if opening_rank is None: + return False + return any(reachable_rank > opening_rank for model in reachable if (reachable_rank := ranks.get(model)) is not None) + + +def _group_by_session(turns: Iterable[Turn]) -> Mapping[str, tuple[Turn, ...]]: + ordered: Final = sorted(turns, key=lambda turn: turn.session_id) + return MappingProxyType( + {session_id: tuple(group) for session_id, group in groupby(ordered, key=lambda turn: turn.session_id)} + ) + + +def signals_for_cohort( + turns: Sequence[Turn], + ranks: Mapping[str, int], + reachable: Iterable[str], +) -> CohortSignals: + """Both rates over the sessions that could have escalated. + + Abandonment shares escalation's denominator on purpose. The two numbers sit next to each + other in the UI and get read as one population; computing them over different sets would + make that reading wrong in a way nothing on screen would reveal. + """ + reachable_models: Final = tuple(reachable) + eligible: Final = tuple( + session_turns + for session_turns in _group_by_session(turns).values() + if could_escalate(session_turns, ranks, reachable_models) + ) + if not eligible: + return CohortSignals(sessions=0, escalation_rate_pct=None, abandonment_rate_pct=None) + + escalated: Final = sum(1 for session_turns in eligible if session_escalated(session_turns, ranks)) + eligible_turns: Final = tuple(turn for session_turns in eligible for turn in session_turns) + abandoned: Final = sum(1 for turn in eligible_turns if turn.client_disconnected) + return CohortSignals( + sessions=len(eligible), + escalation_rate_pct=round(100.0 * escalated / len(eligible), 1), + abandonment_rate_pct=round(100.0 * abandoned / len(eligible_turns), 1) if eligible_turns else None, + ) + + +def baseline_unavailable_reason(turns: Sequence[Turn], cohort: CohortSignals) -> str | None: + """Why the non-routed cohort cannot be shown, or None when it can. + + Session-id coverage is checked before size because the two failures need different words: + a deployment that never sends session ids has plenty of rows and no sessions, and telling + it "not enough traffic" would send it looking for volume it already has. + """ + if turns: + with_client_id: Final = sum(1 for turn in turns if turn.has_client_session_id) + if with_client_id / len(turns) < MIN_SESSION_ID_COVERAGE: + return "no_session_ids" + if cohort.sessions < MIN_COHORT_SESSIONS: + return "insufficient_sessions" + return None diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 141094f4d4c..88e5ae1b36d 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -6,6 +6,7 @@ POST /auto_router/test_routing - Route one prompt through an unsaved complexity- from collections.abc import Sequence from datetime import datetime, timedelta, timezone +from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Final from pydantic import BaseModel, TypeAdapter @@ -34,6 +35,9 @@ from litellm.types.management_endpoints.auto_router_endpoints import ( AutoRouterBenchmarkTotals, AutoRouterCacheBucket, AutoRouterCacheStats, + AutoRouterQualityCohort, + AutoRouterQualitySignals, + AutoRouterQualitySignalsResponse, AutoRouterRoutingTestRequest, AutoRouterRoutingTestResponse, RequestComplexityRouterConfig, @@ -454,3 +458,180 @@ async def get_auto_router_benchmarks( totals=_benchmark_totals(_summed_agg_row(rows)), groups=groups, ) + + +class _QualityTurnRow(BaseModel): + """One request from the window, carrying only what the two signals read.""" + + session_id: str + model: str + started_at: float + router_name: str | None + client_disconnected: bool + session_turn_count: int + api_key: str + + +_QUALITY_TURN_ROWS: Final = TypeAdapter(list[_QualityTurnRow]) + +# Both cohorts come from this one scan, so they cannot drift apart: same window, same +# session grouping, same disconnect test. `router_name` is NULL for a directly-addressed +# request, which is what separates the two populations downstream. +# +# Abandonment reads error_information.error_code rather than `status`, because a client +# disconnect is still billed as a success -- partial streamed spend is deliberately charged +# on disconnect -- so `status` cannot see it. +# +# session_turn_count is how many rows in the window share this session_id. There is no +# persisted flag for "the caller supplied this id rather than the spend writer minting a +# fallback uuid" (`_get_session_id_for_spend_log`), but a minted fallback is always unique to +# its one request, so a session_id that repeats could only have come from the caller. Counting +# it this way needs nothing beyond columns every deployment already writes, prompt storage on +# or off, which a check against proxy_server_request (only persisted when prompt storage is on) +# would not have been. +_QUALITY_SIGNALS_SQL: Final = """ +SELECT + session_id, + model, + EXTRACT(EPOCH FROM "startTime")::float8 AS started_at, + (metadata #>> '{routing_decision,router_model_name}') AS router_name, + COALESCE(metadata #>> '{error_information,error_code}' = '499', false) AS client_disconnected, + COUNT(*) OVER (PARTITION BY session_id)::int AS session_turn_count, + api_key +FROM "LiteLLM_SpendLogs" +WHERE "startTime" >= $1::timestamp AND "startTime" < $2::timestamp + AND session_id IS NOT NULL + AND model <> '' +""" + + +def _quality_signals_for( + turns: Sequence["_QualityTurnRow"], + router_name: str | None, + llm_router: "Router | None", +) -> AutoRouterQualitySignals: + """Signals for one router (or all of them) against that router's own comparable traffic. + + The baseline is drawn from the api_keys that used this router, not from every key on the + deployment. A key that can only reach one model has no escalation to make and would drag + the comparison toward zero for reasons that have nothing to do with routing quality; + restricting to keys that actually used the router keeps both sides of the comparison + inside the same reachable-model world. + + Reachability is judged against every model the key exercised anywhere in the window, not + against the models this one cohort happened to use. Scoping it to the cohort makes a + cohort that never escalated define its own ceiling, so every one of its sessions looks + like it had nowhere to go and the rate reads as unmeasurable rather than as zero -- + silently deleting exactly the well-behaved traffic the comparison exists to show. + """ + from litellm.proxy.db.autorouter_quality_signals import ( + Turn, + baseline_unavailable_reason, + rank_models_by_cost, + signals_for_cohort, + ) + + routed_rows: Final = tuple( + row for row in turns if row.router_name is not None and (router_name is None or row.router_name == router_name) + ) + router_keys: Final = frozenset(row.api_key for row in routed_rows) + baseline_rows: Final = tuple(row for row in turns if row.router_name is None and row.api_key in router_keys) + + reachable: Final = tuple(frozenset(row.model for row in turns if row.api_key in router_keys)) + ranks: Final = rank_models_by_cost(llm_router, reachable) if llm_router is not None else MappingProxyType({}) + + def as_turns(rows: Sequence[_QualityTurnRow]) -> tuple[Turn, ...]: + return tuple( + Turn( + session_id=row.session_id, + model=row.model, + started_at=row.started_at, + client_disconnected=row.client_disconnected, + router_name=row.router_name, + # A fallback uuid (_get_session_id_for_spend_log) is unique to its one + # request; only a caller-supplied session id repeats across rows. + has_client_session_id=row.session_turn_count > 1, + ) + for row in rows + ) + + routed: Final = signals_for_cohort(as_turns(routed_rows), ranks, reachable) + baseline: Final = signals_for_cohort(as_turns(baseline_rows), ranks, reachable) + unavailable: Final = baseline_unavailable_reason(as_turns(baseline_rows), baseline) + return AutoRouterQualitySignals( + router_name=router_name, + routed=AutoRouterQualityCohort( + sessions=routed.sessions, + escalation_rate_pct=routed.escalation_rate_pct, + abandonment_rate_pct=routed.abandonment_rate_pct, + ), + baseline=None + if unavailable + else AutoRouterQualityCohort( + sessions=baseline.sessions, + escalation_rate_pct=baseline.escalation_rate_pct, + abandonment_rate_pct=baseline.abandonment_rate_pct, + ), + baseline_unavailable_reason=unavailable, + ) + + +@router.get( + "/auto_router/quality_signals", + tags=("auto router",), + response_model=AutoRouterQualitySignalsResponse, +) +async def get_auto_router_quality_signals( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + start_date: Annotated[ + str | None, Query(description="YYYY-MM-DD UTC, inclusive (defaults to 30 days before end_date)") + ] = None, + end_date: Annotated[str | None, Query(description="YYYY-MM-DD UTC, inclusive (defaults to today)")] = None, +) -> AutoRouterQualitySignalsResponse: + """ + Quality signals for the auto-router dashboard: how often callers escalated to a costlier + model mid-session, and how often they hung up mid-stream, for auto-routed traffic and for + the same keys' directly-addressed traffic. + + Reads LiteLLM_SpendLogs rather than the per-session rollup, because escalation is a + question about turn order and the rollup folds order away. + + The two cohorts self-select, so this is directional evidence and not an experiment: a + deployment that pins its hardest prompts to one model and routes only the easy ones will + show a flattering baseline. + """ + from litellm.proxy.proxy_server import llm_router, prisma_client + + if user_api_key_dict.user_role not in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ): + raise HTTPException( + status_code=403, + detail="Only proxy admin roles can view auto-router quality signals across the deployment", + ) + if prisma_client is None: + raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) + + end_day: Final = ( + _parse_benchmark_day(end_date) + if end_date + else datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=None) + ) + start_day: Final = _parse_benchmark_day(start_date) if start_date else end_day - timedelta(days=30) + if end_day < start_day: + raise HTTPException(status_code=400, detail="end_date must not be earlier than start_date") + + raw_rows: Final = await prisma_client.db.query_raw( + _QUALITY_SIGNALS_SQL, + start_day.isoformat(), + (end_day + timedelta(days=1)).isoformat(), + ) + turns: Final = _QUALITY_TURN_ROWS.validate_python(raw_rows or ()) + router_names: Final = tuple(sorted(frozenset(row.router_name for row in turns if row.router_name is not None))) + return AutoRouterQualitySignalsResponse( + start_date=start_day.strftime("%Y-%m-%d"), + end_date=end_day.strftime("%Y-%m-%d"), + totals=_quality_signals_for(turns, None, llm_router), + groups=tuple(_quality_signals_for(turns, name, llm_router) for name in router_names), + ) diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 6c8fb96a729..056ff0cc5a2 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -130,3 +130,57 @@ class AutoRouterBenchmarksResponse(BaseModel): routers_in_scope: int totals: AutoRouterBenchmarkTotals groups: tuple[AutoRouterBenchmarkGroup, ...] + + +class AutoRouterQualityCohort(BaseModel): + """One population's quality signals, over sessions that could have escalated. + + Both the routed and the non-routed cohort are measured with the same definitions over + the same table, so the two are directly comparable. ``sessions`` is the denominator for + every rate here: sessions whose key had a strictly more capable model available than the + one the session ran on, since a session that could not escalate cannot evidence a miss. + """ + + sessions: int = Field(description="Sessions in this cohort that had somewhere to escalate to") + escalation_rate_pct: float | None = Field( + description="Share of sessions where a turn moved to a more expensive model than the turn before it" + ) + abandonment_rate_pct: float | None = Field( + description="Share of turns the client disconnected before the stream finished" + ) + + +class AutoRouterQualitySignals(BaseModel): + """Quality signals for one auto-router, against the operator's own comparable traffic. + + Not a controlled experiment: the two cohorts self-select, so a deployment that pins its + hardest prompts to a fixed model and routes only the easy ones will show a flattering + baseline. The comparison is directional evidence, and the UI says so. + """ + + router_name: str | None = Field( + default=None, description="The router these signals cover; None when they cover all routers" + ) + routed: AutoRouterQualityCohort = Field(description="Sessions that went through the auto-router") + baseline: AutoRouterQualityCohort | None = Field( + description="Comparable sessions on a directly-addressed model, or None when there are too few " + "to compare, or too few carry a client-supplied session id to group into real sessions" + ) + baseline_unavailable_reason: str | None = Field( + default=None, + description="Why baseline is None: 'insufficient_sessions' or 'no_session_ids'", + ) + + +class AutoRouterQualitySignalsResponse(BaseModel): + """Quality signals for the auto-router dashboard, computed from per-request spend logs. + + Separate from the benchmarks endpoint because this reads LiteLLM_SpendLogs rather than the + per-session rollup: escalation is an ordered, per-turn question, and the rollup deliberately + folds ordering away. + """ + + start_date: str = Field(description="Window start day, YYYY-MM-DD UTC, inclusive") + end_date: str = Field(description="Window end day, YYYY-MM-DD UTC, inclusive") + totals: AutoRouterQualitySignals = Field(description="Signals over every auto-router in the window") + groups: tuple[AutoRouterQualitySignals, ...] = Field(description="Signals per auto-router") diff --git a/tests/test_litellm/proxy/db/test_autorouter_quality_signals.py b/tests/test_litellm/proxy/db/test_autorouter_quality_signals.py new file mode 100644 index 00000000000..f391d056252 --- /dev/null +++ b/tests/test_litellm/proxy/db/test_autorouter_quality_signals.py @@ -0,0 +1,186 @@ +""" +Unit tests for the auto-router quality signal computation (escalation, abandonment). + +These test the pure logic in litellm.proxy.db.autorouter_quality_signals against +plain Turn objects and a fixed rank table, so they exercise the actual escalation/ +abandonment/eligibility rules without touching Postgres or a real Router. +""" + +from litellm.proxy.db.autorouter_quality_signals import ( + MIN_COHORT_SESSIONS, + MIN_SESSION_ID_COVERAGE, + Turn, + baseline_unavailable_reason, + could_escalate, + session_escalated, + signals_for_cohort, +) + +# haiku < sonnet < opus, matching how rank_models_by_cost would order real deployments +RANKS = {"haiku": 0, "sonnet": 1, "opus": 2} + + +def _turn(session_id="s1", model="sonnet", started_at=0.0, client_disconnected=False, has_client_session_id=True): + return Turn( + session_id=session_id, + model=model, + started_at=started_at, + client_disconnected=client_disconnected, + router_name="auto-router", + has_client_session_id=has_client_session_id, + ) + + +class TestSessionEscalated: + def test_upward_move_is_escalation(self): + turns = [_turn(model="sonnet", started_at=1), _turn(model="opus", started_at=2)] + assert session_escalated(turns, RANKS) is True + + def test_downward_move_is_not_escalation(self): + # This is the exact case the plan explicitly excludes: sonnet -> haiku is a cost + # or latency choice, not a quality miss, and must not be counted. + turns = [_turn(model="sonnet", started_at=1), _turn(model="haiku", started_at=2)] + assert session_escalated(turns, RANKS) is False + + def test_same_model_every_turn_is_not_escalation(self): + turns = [_turn(model="sonnet", started_at=1), _turn(model="sonnet", started_at=2)] + assert session_escalated(turns, RANKS) is False + + def test_single_turn_session_cannot_escalate(self): + assert session_escalated([_turn(model="sonnet", started_at=1)], RANKS) is False + + def test_escalation_detected_regardless_of_input_order(self): + # Turns can arrive from the DB in any order; escalation must be judged on + # started_at, not on list position. + turns = [_turn(model="opus", started_at=2), _turn(model="sonnet", started_at=1)] + assert session_escalated(turns, RANKS) is True + + def test_unrankable_model_does_not_count_as_escalation(self): + turns = [_turn(model="sonnet", started_at=1), _turn(model="mystery-model", started_at=2)] + assert session_escalated(turns, RANKS) is False + + def test_late_escalation_is_still_detected(self): + turns = [ + _turn(model="sonnet", started_at=1), + _turn(model="sonnet", started_at=2), + _turn(model="sonnet", started_at=3), + _turn(model="opus", started_at=4), + ] + assert session_escalated(turns, RANKS) is True + + +class TestCouldEscalate: + def test_session_on_cheapest_model_with_pricier_model_reachable_could_escalate(self): + turns = [_turn(model="haiku")] + assert could_escalate(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) is True + + def test_session_already_on_most_expensive_reachable_model_could_not_escalate(self): + # This is the case the plan calls out: a session pinned to the priciest model it + # could reach had nowhere to go, so it must not silently count as "no miss". + turns = [_turn(model="opus")] + assert could_escalate(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) is False + + def test_reachable_set_without_anything_costlier_cannot_escalate(self): + turns = [_turn(model="sonnet")] + assert could_escalate(turns, RANKS, reachable=["haiku", "sonnet"]) is False + + def test_no_rankable_turns_cannot_escalate(self): + turns = [_turn(model="unknown-model")] + assert could_escalate(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) is False + + def test_eligibility_is_judged_from_the_opening_turn_not_the_ceiling_reached(self): + # A session that started on sonnet and already escalated to opus must still count + # as "could escalate" -- judging from the max model used would make every session + # that actually escalated look ineligible, driving the measured rate toward zero + # exactly as the true rate rises. + turns = [_turn(model="sonnet", started_at=1), _turn(model="opus", started_at=2)] + assert could_escalate(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) is True + + def test_eligibility_uses_first_turn_by_time_not_by_list_order(self): + turns = [_turn(model="opus", started_at=2), _turn(model="sonnet", started_at=1)] + assert could_escalate(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) is True + + +class TestSignalsForCohort: + def test_escalation_rate_counts_only_eligible_sessions(self): + # s1 escalates and could have; s2 sits on opus (the ceiling) so it is excluded from + # the denominator entirely, not counted as a non-escalating session. + turns = [ + _turn(session_id="s1", model="sonnet", started_at=1), + _turn(session_id="s1", model="opus", started_at=2), + _turn(session_id="s2", model="opus", started_at=1), + ] + result = signals_for_cohort(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) + assert result.sessions == 1 + assert result.escalation_rate_pct == 100.0 + + def test_abandonment_counted_over_eligible_turns_only(self): + turns = [ + _turn(session_id="s1", model="haiku", started_at=1, client_disconnected=True), + _turn(session_id="s1", model="haiku", started_at=2, client_disconnected=False), + # s2 is on the ceiling model and ineligible; its disconnect must not be counted. + _turn(session_id="s2", model="opus", started_at=1, client_disconnected=True), + ] + result = signals_for_cohort(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) + assert result.sessions == 1 + assert result.abandonment_rate_pct == 50.0 + + def test_no_eligible_sessions_returns_none_rates_not_zero(self): + # Every session already on the ceiling model: zero would misleadingly claim + # "no miss detected" when in fact nothing could be measured. + turns = [_turn(session_id="s1", model="opus")] + result = signals_for_cohort(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) + assert result.sessions == 0 + assert result.escalation_rate_pct is None + assert result.abandonment_rate_pct is None + + def test_downward_switch_never_inflates_escalation_rate(self): + # Opens on sonnet (eligible: opus is reachable and costlier), then moves down to + # haiku. The downward move must not be read as an escalation. + turns = [ + _turn(session_id="s1", model="sonnet", started_at=1), + _turn(session_id="s1", model="haiku", started_at=2), + ] + result = signals_for_cohort(turns, RANKS, reachable=["haiku", "sonnet", "opus"]) + assert result.sessions == 1 + assert result.escalation_rate_pct == 0.0 + + +class TestBaselineUnavailableReason: + def test_low_session_id_coverage_reported_before_size(self): + turns = [_turn(has_client_session_id=False) for _ in range(100)] + from litellm.proxy.db.autorouter_quality_signals import CohortSignals + + cohort = CohortSignals(sessions=100, escalation_rate_pct=1.0, abandonment_rate_pct=1.0) + assert baseline_unavailable_reason(turns, cohort) == "no_session_ids" + + def test_high_session_id_coverage_with_too_few_sessions_reports_size(self): + from litellm.proxy.db.autorouter_quality_signals import CohortSignals + + turns = [_turn(has_client_session_id=True) for _ in range(10)] + cohort = CohortSignals(sessions=MIN_COHORT_SESSIONS - 1, escalation_rate_pct=1.0, abandonment_rate_pct=1.0) + assert baseline_unavailable_reason(turns, cohort) == "insufficient_sessions" + + def test_sufficient_coverage_and_size_reports_available(self): + from litellm.proxy.db.autorouter_quality_signals import CohortSignals + + turns = [_turn(has_client_session_id=True) for _ in range(10)] + cohort = CohortSignals(sessions=MIN_COHORT_SESSIONS, escalation_rate_pct=1.0, abandonment_rate_pct=1.0) + assert baseline_unavailable_reason(turns, cohort) is None + + def test_coverage_exactly_at_floor_is_available(self): + from litellm.proxy.db.autorouter_quality_signals import CohortSignals + + n = 100 + with_id = int(n * MIN_SESSION_ID_COVERAGE) + turns = [_turn(has_client_session_id=True) for _ in range(with_id)] + [ + _turn(has_client_session_id=False) for _ in range(n - with_id) + ] + cohort = CohortSignals(sessions=MIN_COHORT_SESSIONS, escalation_rate_pct=1.0, abandonment_rate_pct=1.0) + assert baseline_unavailable_reason(turns, cohort) is None + + def test_no_turns_at_all_only_checked_against_size_floor(self): + from litellm.proxy.db.autorouter_quality_signals import CohortSignals + + cohort = CohortSignals(sessions=0, escalation_rate_pct=None, abandonment_rate_pct=None) + assert baseline_unavailable_reason([], cohort) == "insufficient_sessions" diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 888db031515..9c22575f8a5 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -317,9 +317,7 @@ class TestAutoRouterBenchmarks: from litellm.proxy.management_endpoints.auto_router_endpoints import _benchmark_totals totals = _benchmark_totals(self.ROW) - bucket_hits = ( - totals.cache.same_model.hits + totals.cache.first_visit.hits + totals.cache.return_to_tier.hits - ) + bucket_hits = totals.cache.same_model.hits + totals.cache.first_visit.hits + totals.cache.return_to_tier.hits assert bucket_hits == 27 assert totals.cache.hit_rate_pct == pytest.approx(100.0 * 28 / 38, abs=0.1) @@ -427,3 +425,182 @@ class TestAutoRouterBenchmarks: assert response.routers_in_scope == 1 assert response.groups[0].router_name == "live-auto" assert response.groups[0].saved_pct == response.totals.saved_pct == 75.0 + + +class TestAutoRouterQualitySignals: + """The endpoint's cohort split and its refusal to report a baseline it cannot stand behind. + + Escalation/abandonment arithmetic itself is covered in + tests/test_litellm/proxy/db/test_autorouter_quality_signals.py; these tests cover the + wiring around it -- which rows become which cohort, and when the baseline is withheld. + """ + + CHEAP = "openai/gpt-4o-mini" + PRICEY = "openai/gpt-4o" + + @staticmethod + def _row( + session_id: str, + model: str, + started_at: float, + *, + router_name: str | None = "live-auto", + client_disconnected: bool = False, + session_turn_count: int = 2, + api_key: str = "key-1", + ) -> dict: + return { + "session_id": session_id, + "model": model, + "started_at": started_at, + "router_name": router_name, + "client_disconnected": client_disconnected, + "session_turn_count": session_turn_count, + "api_key": api_key, + } + + @staticmethod + def _pricing_router() -> Router: + return Router( + model_list=[ + {"model_name": name, "litellm_params": {"model": name, "api_key": "fake-key"}} + for name in (TestAutoRouterQualitySignals.CHEAP, TestAutoRouterQualitySignals.PRICEY) + ] + ) + + async def _call(self, rows: list[dict], monkeypatch: pytest.MonkeyPatch, **kwargs: object): + from litellm.proxy import proxy_server + from litellm.proxy.management_endpoints.auto_router_endpoints import ( + get_auto_router_quality_signals, + ) + + class _DB: + async def query_raw(self, sql: str, *params: object): + return rows + + monkeypatch.setattr(proxy_server, "prisma_client", type("P", (), {"db": _DB()})()) + monkeypatch.setattr(proxy_server, "llm_router", self._pricing_router()) + return await get_auto_router_quality_signals( + user_api_key_dict=ADMIN, + start_date="2026-08-01", + end_date="2026-08-02", + **kwargs, + ) + + @pytest.mark.asyncio + async def test_non_admin_roles_cannot_read_quality_signals(self): + from litellm.proxy.management_endpoints.auto_router_endpoints import ( + get_auto_router_quality_signals, + ) + + with pytest.raises(HTTPException) as err: + await get_auto_router_quality_signals( + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="sk-x"), + start_date="2026-08-01", + end_date="2026-08-02", + ) + assert err.value.status_code == 403 + + @pytest.mark.asyncio + async def test_a_reversed_window_is_rejected(self, monkeypatch: pytest.MonkeyPatch): + from litellm.proxy import proxy_server + from litellm.proxy.management_endpoints.auto_router_endpoints import ( + get_auto_router_quality_signals, + ) + + monkeypatch.setattr(proxy_server, "prisma_client", object()) + with pytest.raises(HTTPException) as err: + await get_auto_router_quality_signals( + user_api_key_dict=ADMIN, start_date="2026-08-05", end_date="2026-08-01" + ) + assert err.value.status_code == 400 + + @pytest.mark.asyncio + async def test_routed_escalation_is_measured_from_routed_rows(self, monkeypatch: pytest.MonkeyPatch): + rows = [ + self._row("s1", self.CHEAP, 1.0), + self._row("s1", self.PRICEY, 2.0), + ] + response = await self._call(rows, monkeypatch) + assert response.totals.routed.sessions == 1 + assert response.totals.routed.escalation_rate_pct == 100.0 + + @pytest.mark.asyncio + async def test_rows_without_a_routing_decision_are_the_baseline_not_the_routed_cohort( + self, monkeypatch: pytest.MonkeyPatch + ): + # Same key, same models; only the presence of a router name separates the cohorts. + # The routed session escalates and the direct one does not, so a cohort mix-up + # cannot produce these two numbers. + rows = [ + self._row("routed-1", self.CHEAP, 1.0), + self._row("routed-1", self.PRICEY, 2.0), + ] + rows += [self._row(f"direct-{i}", self.CHEAP, 1.0, router_name=None) for i in range(30)] + rows += [self._row(f"direct-{i}", self.CHEAP, 2.0, router_name=None) for i in range(30)] + response = await self._call(rows, monkeypatch) + assert response.totals.routed.escalation_rate_pct == 100.0 + assert response.totals.baseline is not None + assert response.totals.baseline.sessions == 30 + assert response.totals.baseline.escalation_rate_pct == 0.0 + + @pytest.mark.asyncio + async def test_baseline_withheld_when_sessions_are_mostly_one_request_fallback_ids( + self, monkeypatch: pytest.MonkeyPatch + ): + # session_turn_count == 1 marks a session id the spend writer minted rather than one + # the caller supplied. A cohort made of those is not sessions, and reporting a rate + # over it would describe the fallback rather than the traffic. + rows = [self._row("routed-1", self.CHEAP, 1.0), self._row("routed-1", self.PRICEY, 2.0)] + rows += [self._row(f"direct-{i}", self.CHEAP, 1.0, router_name=None, session_turn_count=1) for i in range(50)] + response = await self._call(rows, monkeypatch) + assert response.totals.baseline is None + assert response.totals.baseline_unavailable_reason == "no_session_ids" + + @pytest.mark.asyncio + async def test_baseline_withheld_when_there_is_too_little_comparable_traffic(self, monkeypatch: pytest.MonkeyPatch): + rows = [self._row("routed-1", self.CHEAP, 1.0), self._row("routed-1", self.PRICEY, 2.0)] + rows += [ + self._row("direct-1", self.CHEAP, 1.0, router_name=None), + self._row("direct-1", self.CHEAP, 2.0, router_name=None), + ] + response = await self._call(rows, monkeypatch) + assert response.totals.baseline is None + assert response.totals.baseline_unavailable_reason == "insufficient_sessions" + + @pytest.mark.asyncio + async def test_baseline_only_draws_on_keys_that_used_the_router(self, monkeypatch: pytest.MonkeyPatch): + # A key that never touched the router says nothing about the router, and its traffic + # must not be folded into the comparison. + rows = [self._row("routed-1", self.CHEAP, 1.0), self._row("routed-1", self.PRICEY, 2.0)] + rows += [self._row(f"other-{i}", self.CHEAP, 1.0, router_name=None, api_key="unrelated-key") for i in range(50)] + response = await self._call(rows, monkeypatch) + assert response.totals.baseline is None + + @pytest.mark.asyncio + async def test_groups_are_reported_per_router(self, monkeypatch: pytest.MonkeyPatch): + rows = [ + self._row("a1", self.CHEAP, 1.0, router_name="router-a"), + self._row("a1", self.PRICEY, 2.0, router_name="router-a"), + self._row("b1", self.CHEAP, 1.0, router_name="router-b"), + self._row("b1", self.CHEAP, 2.0, router_name="router-b"), + ] + response = await self._call(rows, monkeypatch) + by_name = {group.router_name: group for group in response.groups} + assert set(by_name) == {"router-a", "router-b"} + assert by_name["router-a"].routed.escalation_rate_pct == 100.0 + assert by_name["router-b"].routed.escalation_rate_pct == 0.0 + + @pytest.mark.asyncio + async def test_abandonment_reads_the_disconnect_flag_not_the_status(self, monkeypatch: pytest.MonkeyPatch): + # s2 gives the key a pricier model in the window, so s1 is eligible; without it s1 + # sits at its own ceiling and is correctly excluded, measuring nothing. + rows = [ + self._row("s1", self.CHEAP, 1.0, client_disconnected=True), + self._row("s1", self.CHEAP, 2.0, client_disconnected=False), + self._row("s2", self.PRICEY, 1.0), + self._row("s2", self.PRICEY, 2.0), + ] + response = await self._call(rows, monkeypatch) + assert response.totals.routed.sessions == 1 + assert response.totals.routed.abandonment_rate_pct == 50.0 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx index a5767383307..1806b748fb4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx @@ -1,10 +1,11 @@ import { fireEvent, render, screen } from "@testing-library/react"; import React from "react"; -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import { ApiError } from "@/lib/http/client"; vi.mock("./useAutoRouterBenchmarks", () => ({ useAutoRouterBenchmarks: vi.fn() })); +vi.mock("./useAutoRouterQualitySignals", () => ({ useAutoRouterQualitySignals: vi.fn() })); import AutoRouterBenchmarksTab from "./AutoRouterBenchmarksTab"; import type { @@ -12,9 +13,12 @@ import type { AutoRouterBenchmarksResponse, AutoRouterCacheStats, } from "./autoRouterBenchmarks"; +import type { AutoRouterQualitySignalsResponse } from "./autoRouterQualitySignals"; import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks"; +import { useAutoRouterQualitySignals } from "./useAutoRouterQualitySignals"; type HookResult = ReturnType; +type QualityHookResult = ReturnType; const mockHook = (result: { data?: AutoRouterBenchmarksResponse; isPending?: boolean; error?: Error }) => { vi.mocked(useAutoRouterBenchmarks).mockReturnValue({ @@ -24,6 +28,14 @@ const mockHook = (result: { data?: AutoRouterBenchmarksResponse; isPending?: boo } as unknown as HookResult); }; +const mockQualityHook = (data?: AutoRouterQualitySignalsResponse) => { + vi.mocked(useAutoRouterQualitySignals).mockReturnValue({ + data, + isPending: false, + error: null, + } as unknown as QualityHookResult); +}; + const cache = (overrides: Partial = {}): AutoRouterCacheStats => ({ coverage_pct: 99.6, hit_rate_pct: 93.3, @@ -73,7 +85,35 @@ const response = (groups: AutoRouterBenchmarkGroup[], shared: Totals = totals()) const renderTab = () => render(); +const cohort = ( + overrides: Partial = {}, +): AutoRouterQualitySignalsResponse["totals"]["routed"] => ({ + sessions: 120, + escalation_rate_pct: 6.9, + abandonment_rate_pct: 2.4, + ...overrides, +}); + +const qualityResponse = ( + overrides: Partial = {}, +): AutoRouterQualitySignalsResponse => ({ + start_date: "2026-07-06", + end_date: "2026-08-05", + totals: { + router_name: null, + routed: cohort(), + baseline: cohort({ sessions: 80, escalation_rate_pct: 3.1, abandonment_rate_pct: 2.1 }), + baseline_unavailable_reason: null, + ...overrides, + }, + groups: [], +}); + describe("AutoRouterBenchmarksTab", () => { + beforeEach(() => { + mockQualityHook(undefined); + }); + it("leads with total estimated savings, before the three session-shape metrics", () => { mockHook({ data: response([group(), group({ router_name: "gpt-auto" })]) }); renderTab(); @@ -264,4 +304,69 @@ describe("AutoRouterBenchmarksTab", () => { expect(screen.getByRole("tab", { name: "30d" })).toBeInTheDocument(); expect(screen.getByText("All auto-routers")).toBeInTheDocument(); }); + + describe("quality signals", () => { + it("renders escalation and abandonment against the non-router baseline", () => { + mockHook({ data: response([group()]) }); + mockQualityHook(qualityResponse()); + renderTab(); + + expect(screen.getByText("Escalation rate")).toBeInTheDocument(); + expect(screen.getByText("6.9%")).toBeInTheDocument(); + expect(screen.getByText("vs. 3.1% on your non-router traffic")).toBeInTheDocument(); + + expect(screen.getByText("Stream abandonment")).toBeInTheDocument(); + expect(screen.getByText("2.4%")).toBeInTheDocument(); + expect(screen.getByText("vs. 2.1% on your non-router traffic")).toBeInTheDocument(); + }); + + it("does not render the quality card when there is no quality data yet", () => { + mockHook({ data: response([group()]) }); + mockQualityHook(undefined); + renderTab(); + + expect(screen.queryByText("Escalation rate")).not.toBeInTheDocument(); + }); + + it("explains why the baseline is missing instead of showing a misleading rate", () => { + mockHook({ data: response([group()]) }); + mockQualityHook( + qualityResponse({ + baseline: null, + baseline_unavailable_reason: "no_session_ids", + }), + ); + renderTab(); + + expect( + screen.getAllByText("Non-router traffic isn't sending session IDs, so it can't be grouped into sessions to compare"), + ).toHaveLength(2); + }); + + it("explains an insufficient-sessions baseline distinctly from a missing-session-id one", () => { + mockHook({ data: response([group()]) }); + mockQualityHook( + qualityResponse({ + baseline: null, + baseline_unavailable_reason: "insufficient_sessions", + }), + ); + renderTab(); + + expect(screen.getAllByText("Not enough comparable non-router traffic in this window to compare")).toHaveLength(2); + }); + + it("flags escalation as worse than baseline visually distinctly from a healthy rate", () => { + mockHook({ data: response([group()]) }); + mockQualityHook( + qualityResponse({ + routed: cohort({ escalation_rate_pct: 9.0 }), + baseline: cohort({ escalation_rate_pct: 3.0 }), + }), + ); + renderTab(); + + expect(screen.getByText("9.0%")).toHaveClass("text-destructive"); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx index ff0f52940b2..547b44ed966 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx @@ -28,8 +28,16 @@ import { type BenchmarkWindow, type BucketRow, } from "./autoRouterBenchmarks"; +import { + BASELINE_UNAVAILABLE_COPY, + ratePctLabel, + signalsFor, + type AutoRouterQualitySignals, + type AutoRouterQualitySignalsResponse, +} from "./autoRouterQualitySignals"; import { usd } from "./costOptimizationUtils"; import { useAutoRouterBenchmarks } from "./useAutoRouterBenchmarks"; +import { useAutoRouterQualitySignals } from "./useAutoRouterQualitySignals"; const Message: React.FC<{ children: React.ReactNode }> = ({ children }) => (

{children}

@@ -64,6 +72,10 @@ const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => { {Math.abs(stats.saved_pct).toFixed(0)}% +
+

Avg saved per session

+

{usd(stats.saved_per_session)}

+
@@ -90,18 +102,72 @@ const HeroCard: React.FC<{ view: BenchmarkView }> = ({ view }) => {

{stats.turns.toLocaleString()}

-
-
-
Avg saved per session
-
{usd(stats.saved_per_session)}
-
-
); }; +const QualityMetric: React.FC<{ + label: string; + hint: string; + routed: number | null | undefined; + baseline: number | null | undefined; + baselineUnavailableReason: string | null | undefined; +}> = ({ label, hint, routed, baseline, baselineUnavailableReason }) => { + const worseThanBaseline = routed != null && baseline != null && routed > baseline; + return ( +
+ + + + {label} + + } + /> + {hint} + + +

+ {ratePctLabel(routed)} +

+

+ {baseline == null + ? (baselineUnavailableReason && BASELINE_UNAVAILABLE_COPY[baselineUnavailableReason]) || + "No comparable non-router traffic" + : `vs. ${ratePctLabel(baseline)} on your non-router traffic`} +

+
+ ); +}; + +const QualityCard: React.FC<{ signals: AutoRouterQualitySignals }> = ({ signals }) => ( + +

Quality signals

+ + + +); + const StackedTurnBar: React.FC<{ buckets: BucketRow[] }> = ({ buckets }) => { const segments = buckets.filter((b) => b.turns > 0); return ( @@ -233,9 +299,10 @@ interface BenchmarksBodyProps { error: unknown; data: AutoRouterBenchmarksResponse | undefined; selectedKey: string; + qualityData: AutoRouterQualitySignalsResponse | undefined; } -const BenchmarksBody: React.FC = ({ isPending, error, data, selectedKey }) => { +const BenchmarksBody: React.FC = ({ isPending, error, data, selectedKey, qualityData }) => { if (isPending) return Loading auto-router usage...; if (error instanceof ApiError && error.status === 403) { return Auto-router usage is visible to proxy admin roles only; @@ -245,9 +312,24 @@ const BenchmarksBody: React.FC = ({ isPending, error, data, const view = viewFor(data, selectedKey); const stats = view.stats; + const selectedGroup = data.groups.find((g) => groupKey(g) === selectedKey); + const signals = qualityData + ? signalsFor(qualityData, selectedKey === ALL_ROUTERS ? null : (selectedGroup?.router_name ?? null)) + : undefined; return ( <> - +
+ + {signals ? : null} +
+ + {signals ? ( +

+ Quality signals compare this router against the same keys' directly-addressed traffic. The two + populations self-select, so this is directional evidence rather than a controlled experiment: routing only + your easier prompts will flatter the comparison. +

+ ) : null}
@@ -281,6 +363,7 @@ interface AutoRouterBenchmarksTabProps { const AutoRouterBenchmarksTab: React.FC = ({ accessToken }) => { const [range, setRange] = useState("30d"); const { data, isPending, error } = useAutoRouterBenchmarks(accessToken, range); + const { data: qualityData } = useAutoRouterQualitySignals(accessToken, range); const [selectedKey, setSelectedKey] = useState(ALL_ROUTERS); const groups = data?.groups ?? []; @@ -319,7 +402,13 @@ const AutoRouterBenchmarksTab: React.FC = ({ acces
- + ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterQualitySignals.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterQualitySignals.ts new file mode 100644 index 00000000000..c0d8b1fd1ba --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/autoRouterQualitySignals.ts @@ -0,0 +1,33 @@ +import type { components } from "@/lib/http/schema"; + +import { ALL_ROUTERS } from "./autoRouterBenchmarks"; + +export type AutoRouterQualitySignalsResponse = components["schemas"]["AutoRouterQualitySignalsResponse"]; +export type AutoRouterQualitySignals = components["schemas"]["AutoRouterQualitySignals"]; +export type AutoRouterQualityCohort = components["schemas"]["AutoRouterQualityCohort"]; + +export const BASELINE_UNAVAILABLE_COPY: Record = { + no_session_ids: "Non-router traffic isn't sending session IDs, so it can't be grouped into sessions to compare", + insufficient_sessions: "Not enough comparable non-router traffic in this window to compare", +}; + +export const signalsFor = ( + data: AutoRouterQualitySignalsResponse, + selectedRouterName: string | null, +): AutoRouterQualitySignals => { + if (selectedRouterName === null || selectedRouterName === ALL_ROUTERS) return data.totals; + return data.groups.find((group) => group.router_name === selectedRouterName) ?? data.totals; +}; + +export const ratePctLabel = (value: number | null | undefined): string => + value === null || value === undefined ? "—" : `${value.toFixed(1)}%`; + +/** Positive when the router escalates more often than the operator's own direct traffic. */ +export const deltaVsBaseline = ( + routed: number | null | undefined, + baseline: number | null | undefined, +): number | null => { + if (routed === null || routed === undefined) return null; + if (baseline === null || baseline === undefined) return null; + return routed - baseline; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterQualitySignals.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterQualitySignals.ts new file mode 100644 index 00000000000..5e8a0cc445b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/useAutoRouterQualitySignals.ts @@ -0,0 +1,11 @@ +import { $api } from "@/lib/http/api"; + +import { windowFor, type BenchmarkWindow } from "./autoRouterBenchmarks"; + +export const useAutoRouterQualitySignals = (accessToken: string | null, range: BenchmarkWindow) => + $api.useQuery( + "get", + "/auto_router/quality_signals", + { params: { query: windowFor(range, new Date()) } }, + { enabled: Boolean(accessToken), retry: false }, + ); diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f1660e77ad9..0fcf206fb08 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -807,6 +807,35 @@ export interface paths { patch?: never; trace?: never; }; + "/auto_router/quality_signals": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Auto Router Quality Signals + * @description Quality signals for the auto-router dashboard: how often callers escalated to a costlier + * model mid-session, and how often they hung up mid-stream, for auto-routed traffic and for + * the same keys' directly-addressed traffic. + * + * Reads LiteLLM_SpendLogs rather than the per-session rollup, because escalation is a + * question about turn order and the rollup folds order away. + * + * The two cohorts self-select, so this is directional evidence and not an experiment: a + * deployment that pins its hardest prompts to one model and routes only the easy ones will + * show a flattering baseline. + */ + get: operations["get_auto_router_quality_signals_auto_router_quality_signals_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/auto_router/test_routing": { parameters: { query?: never; @@ -21539,6 +21568,83 @@ export interface components { /** System Prompt */ system_prompt: string; }; + /** + * AutoRouterQualityCohort + * @description One population's quality signals, over sessions that could have escalated. + * + * Both the routed and the non-routed cohort are measured with the same definitions over + * the same table, so the two are directly comparable. ``sessions`` is the denominator for + * every rate here: sessions whose key had a strictly more capable model available than the + * one the session ran on, since a session that could not escalate cannot evidence a miss. + */ + AutoRouterQualityCohort: { + /** + * Abandonment Rate Pct + * @description Share of turns the client disconnected before the stream finished + */ + abandonment_rate_pct: number | null; + /** + * Escalation Rate Pct + * @description Share of sessions where a turn moved to a more expensive model than the turn before it + */ + escalation_rate_pct: number | null; + /** + * Sessions + * @description Sessions in this cohort that had somewhere to escalate to + */ + sessions: number; + }; + /** + * AutoRouterQualitySignals + * @description Quality signals for one auto-router, against the operator's own comparable traffic. + * + * Not a controlled experiment: the two cohorts self-select, so a deployment that pins its + * hardest prompts to a fixed model and routes only the easy ones will show a flattering + * baseline. The comparison is directional evidence, and the UI says so. + */ + AutoRouterQualitySignals: { + /** @description Comparable sessions on a directly-addressed model, or None when there are too few to compare, or too few carry a client-supplied session id to group into real sessions */ + baseline: components["schemas"]["AutoRouterQualityCohort"] | null; + /** + * Baseline Unavailable Reason + * @description Why baseline is None: 'insufficient_sessions' or 'no_session_ids' + */ + baseline_unavailable_reason?: string | null; + /** @description Sessions that went through the auto-router */ + routed: components["schemas"]["AutoRouterQualityCohort"]; + /** + * Router Name + * @description The router these signals cover; None when they cover all routers + */ + router_name?: string | null; + }; + /** + * AutoRouterQualitySignalsResponse + * @description Quality signals for the auto-router dashboard, computed from per-request spend logs. + * + * Separate from the benchmarks endpoint because this reads LiteLLM_SpendLogs rather than the + * per-session rollup: escalation is an ordered, per-turn question, and the rollup deliberately + * folds ordering away. + */ + AutoRouterQualitySignalsResponse: { + /** + * End Date + * @description Window end day, YYYY-MM-DD UTC, inclusive + */ + end_date: string; + /** + * Groups + * @description Signals per auto-router + */ + groups: components["schemas"]["AutoRouterQualitySignals"][]; + /** + * Start Date + * @description Window start day, YYYY-MM-DD UTC, inclusive + */ + start_date: string; + /** @description Signals over every auto-router in the window */ + totals: components["schemas"]["AutoRouterQualitySignals"]; + }; /** * AutoRouterRoutingTestRequest * @description A single prompt to classify against a complexity-router config that need not be saved yet. @@ -36797,6 +36903,40 @@ export interface operations { }; }; }; + get_auto_router_quality_signals_auto_router_quality_signals_get: { + parameters: { + query?: { + /** @description YYYY-MM-DD UTC, inclusive (defaults to 30 days before end_date) */ + start_date?: string | null; + /** @description YYYY-MM-DD UTC, inclusive (defaults to today) */ + end_date?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AutoRouterQualitySignalsResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; preview_auto_router_routing_auto_router_test_routing_post: { parameters: { query?: never;