diff --git a/litellm/proxy/db/autorouter_quality_signals.py b/litellm/proxy/db/autorouter_quality_signals.py index 36a31d6aaf8..f9663076792 100644 --- a/litellm/proxy/db/autorouter_quality_signals.py +++ b/litellm/proxy/db/autorouter_quality_signals.py @@ -9,9 +9,12 @@ both are already recorded on every request: do the job. High precision, low recall: it only sees callers who *can* switch and bother to, so an API-only integration can report zero while suffering. -``abandonment`` the caller hung up before the stream finished. Low precision, high recall: - it catches the giving-up that escalation structurally misses, but a dropped - connection and "I read enough" look the same from here. +``abandonment`` the caller hung up after content started streaming but before it finished. + Low precision, high recall: it catches the giving-up that escalation + structurally misses, but a dropped connection and "I read enough" look the + same from here. Disconnects before any content arrived are excluded -- + those are a latency or connectivity event, not a judgment on the response, + since the caller never saw one. They are reported side by side rather than blended, because the two fail in opposite directions and one number would hide which of them fired. @@ -46,13 +49,16 @@ MIN_SESSION_ID_COVERAGE: Final = 0.8 class Turn: """One request, reduced to what the two signals need. - ``escalation`` needs the model and the order; ``abandonment`` needs the disconnect flag. + ``escalation`` needs the model and the order; ``abandonment`` needs the disconnect flag + plus ``completion_tokens``, since a disconnect that delivered nothing is not the same + event as one that cut off a response in progress -- see ``was_abandoned``. Kept as a plain object rather than a row dict so the signal functions state their inputs. """ __slots__ = ( "api_key", "client_disconnected", + "completion_tokens", "has_client_session_id", "model", "router_name", @@ -68,6 +74,7 @@ class Turn: model: str, started_at: float, client_disconnected: bool, + completion_tokens: int, router_name: str | None, has_client_session_id: bool, ) -> None: @@ -76,9 +83,20 @@ class Turn: self.model = model self.started_at = started_at self.client_disconnected = client_disconnected + self.completion_tokens = completion_tokens self.router_name = router_name self.has_client_session_id = has_client_session_id + @property + def was_abandoned(self) -> bool: + """True for a disconnect that cut off a response already in progress. + + A disconnect with zero completion tokens delivered nothing for the caller to judge -- + that is a latency or connectivity failure, not evidence about response quality, so it + is excluded from the abandonment signal even though ``client_disconnected`` is True. + """ + return self.client_disconnected and self.completion_tokens > 0 + class CohortSignals: """What one population evidences, and how much of it there was to look at.""" @@ -191,7 +209,7 @@ def signals_for_cohort( escalated: Final = sum(1 for session_turns in eligible if session_escalated(session_turns, ranks)) eligible_turns: Final = tuple(turn for session_turns in eligible for turn in session_turns) - abandoned: Final = sum(1 for turn in eligible_turns if turn.client_disconnected) + abandoned: Final = sum(1 for turn in eligible_turns if turn.was_abandoned) return CohortSignals( sessions=len(eligible), escalation_rate_pct=round(100.0 * escalated / len(eligible), 1), diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 39c84b9c566..54d602bc3e1 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -489,12 +489,15 @@ class _QualityTurnRow(BaseModel): started_at: float router_name: str | None client_disconnected: bool + completion_tokens: int session_turn_count: int api_key: str _QUALITY_TURN_ROWS: Final = TypeAdapter(list[_QualityTurnRow]) +MAX_QUALITY_SIGNAL_ROWS: Final = 100_000 + _QUALITY_SIGNALS_SQL: Final = """ SELECT session_id, @@ -502,12 +505,14 @@ SELECT EXTRACT(EPOCH FROM "startTime")::float8 AS started_at, (metadata #>> '{routing_decision,router_model_name}') AS router_name, COALESCE(metadata #>> '{error_information,error_code}' = '499', false) AS client_disconnected, + completion_tokens, COUNT(*) OVER (PARTITION BY api_key, session_id)::int AS session_turn_count, api_key FROM "LiteLLM_SpendLogs" WHERE "startTime" >= $1::timestamp AND "startTime" < $2::timestamp AND session_id IS NOT NULL AND model <> '' +LIMIT $3 """ @@ -564,6 +569,7 @@ def _quality_signals_for( model=row.model, started_at=row.started_at, client_disconnected=row.client_disconnected, + completion_tokens=row.completion_tokens, router_name=row.router_name, has_client_session_id=row.session_turn_count > 1, ) @@ -650,7 +656,17 @@ async def get_auto_router_quality_signals( _QUALITY_SIGNALS_SQL, start_day.isoformat(), (end_day + timedelta(days=1)).isoformat(), + MAX_QUALITY_SIGNAL_ROWS + 1, ) + if raw_rows is not None and len(raw_rows) > MAX_QUALITY_SIGNAL_ROWS: + raise HTTPException( + status_code=400, + detail=( + f"Window contains more than {MAX_QUALITY_SIGNAL_ROWS:,} session-bearing requests; " + "narrow the date range. Truncating would silently drop the turns these signals " + "are computed from." + ), + ) turns: Final = _QUALITY_TURN_ROWS.validate_python(raw_rows or ()) router_names: Final = tuple(sorted(frozenset(row.router_name for row in turns if row.router_name is not None))) return AutoRouterQualitySignalsResponse( diff --git a/tests/test_litellm/proxy/db/test_autorouter_quality_signals.py b/tests/test_litellm/proxy/db/test_autorouter_quality_signals.py index dbad2433d45..a8b9da4e22a 100644 --- a/tests/test_litellm/proxy/db/test_autorouter_quality_signals.py +++ b/tests/test_litellm/proxy/db/test_autorouter_quality_signals.py @@ -29,6 +29,7 @@ def _turn( model="sonnet", started_at=0.0, client_disconnected=False, + completion_tokens=10, has_client_session_id=True, api_key="key1", ): @@ -38,11 +39,23 @@ def _turn( model=model, started_at=started_at, client_disconnected=client_disconnected, + completion_tokens=completion_tokens, router_name="auto-router", has_client_session_id=has_client_session_id, ) +class TestTurnWasAbandoned: + def test_disconnect_with_delivered_tokens_is_abandoned(self): + assert _turn(client_disconnected=True, completion_tokens=1).was_abandoned is True + + def test_disconnect_with_zero_tokens_is_not_abandoned(self): + assert _turn(client_disconnected=True, completion_tokens=0).was_abandoned is False + + def test_completed_turn_with_zero_tokens_is_not_abandoned(self): + assert _turn(client_disconnected=False, completion_tokens=0).was_abandoned is False + + class TestRankModelsByCost: def test_equal_cost_models_receive_equal_rank(self, monkeypatch: pytest.MonkeyPatch): import litellm.proxy.db.autorouter_quality_signals as module @@ -151,6 +164,26 @@ class TestSignalsForCohort: assert result.sessions == 1 assert result.abandonment_rate_pct == 50.0 + def test_disconnect_before_first_token_is_not_counted_as_abandonment(self): + # A disconnect that delivered zero completion tokens never showed the caller a + # response to judge -- it's a latency/connectivity event, not quality evidence. + turns = [ + _turn(session_id="s1", model="haiku", started_at=1, client_disconnected=True, completion_tokens=0), + _turn(session_id="s1", model="haiku", started_at=2, client_disconnected=False), + _turn( + session_id="s2", + api_key="key1", + model="haiku", + started_at=1, + client_disconnected=True, + completion_tokens=5, + ), + _turn(session_id="s2", api_key="key1", model="haiku", started_at=2, client_disconnected=False), + ] + result = signals_for_cohort(turns, RANKS, REACHABLE_BY_KEY) + assert result.sessions == 2 + assert result.abandonment_rate_pct == 25.0 + def test_no_eligible_sessions_returns_none_rates_not_zero(self): # Every session already on the ceiling model: zero would misleadingly claim # "no miss detected" when in fact nothing could be measured. diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 136c1dc15ba..20bb9b8d82a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -485,6 +485,7 @@ class TestAutoRouterQualitySignals: *, router_name: str | None = "live-auto", client_disconnected: bool = False, + completion_tokens: int = 10, session_turn_count: int = 2, api_key: str = "key-1", ) -> dict: @@ -494,6 +495,7 @@ class TestAutoRouterQualitySignals: "started_at": started_at, "router_name": router_name, "client_disconnected": client_disconnected, + "completion_tokens": completion_tokens, "session_turn_count": session_turn_count, "api_key": api_key, } @@ -554,6 +556,20 @@ class TestAutoRouterQualitySignals: ) assert err.value.status_code == 400 + @pytest.mark.asyncio + async def test_a_window_over_the_row_cap_is_rejected_not_silently_truncated( + self, monkeypatch: pytest.MonkeyPatch + ): + from litellm.proxy.management_endpoints.auto_router_endpoints import ( + MAX_QUALITY_SIGNAL_ROWS, + ) + + rows = [self._row(f"s{i}", self.CHEAP, float(i)) for i in range(MAX_QUALITY_SIGNAL_ROWS + 1)] + with pytest.raises(HTTPException) as err: + await self._call(rows, monkeypatch) + assert err.value.status_code == 400 + assert "narrow the date range" in err.value.detail + @pytest.mark.asyncio async def test_routed_escalation_is_measured_from_routed_rows(self, monkeypatch: pytest.MonkeyPatch): rows = [ @@ -677,3 +693,19 @@ class TestAutoRouterQualitySignals: response = await self._call(rows, monkeypatch) assert response.totals.routed.sessions == 1 assert response.totals.routed.abandonment_rate_pct == 50.0 + + @pytest.mark.asyncio + async def test_disconnect_before_first_token_is_excluded_from_abandonment( + self, monkeypatch: pytest.MonkeyPatch + ): + # Same shape as the test above, but the disconnect delivered nothing: the caller + # never saw a response to judge, so it must not read as abandonment. + rows = [ + self._row("s1", self.CHEAP, 1.0, client_disconnected=True, completion_tokens=0), + self._row("s1", self.CHEAP, 2.0, client_disconnected=False), + self._row("s2", self.PRICEY, 1.0), + self._row("s2", self.PRICEY, 2.0), + ] + response = await self._call(rows, monkeypatch) + assert response.totals.routed.sessions == 1 + assert response.totals.routed.abandonment_rate_pct == 0.0 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx index dfee80fd83c..c825eb9728a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.test.tsx @@ -358,7 +358,7 @@ describe("AutoRouterBenchmarksTab", () => { expect(screen.getAllByText("Not enough comparable non-router traffic in this window to compare")).toHaveLength(2); }); - it("flags escalation as worse than baseline visually distinctly from a healthy rate", () => { + it("renders escalation and abandonment rates without flagging them as regressions", () => { mockHook({ data: response([group()]) }); mockQualityHook( qualityResponse({ @@ -368,7 +368,8 @@ describe("AutoRouterBenchmarksTab", () => { ); renderTab(); - expect(screen.getByText("9.0%")).toHaveClass("text-destructive"); + expect(screen.getByText("9.0%")).toHaveClass("text-foreground"); + expect(screen.getByText("9.0%")).not.toHaveClass("text-destructive"); }); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx index 5263e6bca8b..bd34dc401ac 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-optimization/_components/AutoRouterBenchmarksTab.tsx @@ -115,7 +115,6 @@ const QualityMetric: React.FC<{ baseline: number | null | undefined; baselineUnavailableReason: string | null | undefined; }> = ({ label, hint, routed, baseline, baselineUnavailableReason }) => { - const worseThanBaseline = routed != null && baseline != null && routed > baseline; return (
@@ -133,11 +132,7 @@ const QualityMetric: React.FC<{ {hint} -

- {ratePctLabel(routed)} -

+

{ratePctLabel(routed)}

{baseline == null ? (baselineUnavailableReason && BASELINE_UNAVAILABLE_COPY[baselineUnavailableReason]) || @@ -159,8 +154,8 @@ const QualityCard: React.FC<{ signals: AutoRouterQualitySignals }> = ({ signals baselineUnavailableReason={signals.baseline_unavailable_reason} />