From 26e47aea32636c2ffd98e2b34047a06c22fab0b6 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Mon, 24 Aug 2026 15:28:20 -0400 Subject: [PATCH] fix(auto-router): list configured auto-routers in the usage picker before they have traffic --- .../auto_router_endpoints.py | 62 +++++++- .../auto_router_endpoints.py | 13 +- .../test_auto_router_endpoints.py | 147 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 15 +- 4 files changed, 230 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 46aac82473c..1322f50d4af 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -36,6 +36,7 @@ from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.repositories.base_repository import SupportsModelDump from litellm.repositories.team_repository import TeamRepository from litellm.router_strategy.complexity_router import ComplexityRouter +from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model from litellm.types.management_endpoints.auto_router_endpoints import ( SHADOW_EVAL_TURN_VALVE, AutoRouterBenchmarkGroup, @@ -510,6 +511,53 @@ def _summed_agg_row(rows: Sequence[_SessionAggRow]) -> _SessionAggRow: ) +def _strategy_router_key(deployment: object) -> tuple[str, str] | None: + """``(model_name, kind)`` for a deployment whose routing the session rollup records. + + Kinds come from ``classify_strategy_router_model``, the same rule the Router registers a + deployment by, so this arm cannot disagree with the arm that stamped ``router_type`` onto + the session rows. Semantic auto-routers return None: they record no routing decision, so + they can never own a session row, and ``AutoRouterBenchmarkGroup.router_type`` has no + value for them. A permanent zero would read as "no traffic" rather than "not instrumented". + """ + if not isinstance(deployment, Mapping): + return None + litellm_params: Final = deployment.get("litellm_params") + router_name: Final = deployment.get("model_name") + if not (isinstance(litellm_params, Mapping) and isinstance(router_name, str) and router_name): + return None + model: Final = litellm_params.get("model") + if not isinstance(model, str): + return None + kind: Final = classify_strategy_router_model(model) + return None if kind is None or kind == "semantic" else (router_name, kind) + + +def _idle_router_groups( + llm_router: "Router | None", covered: frozenset[tuple[str, str]] +) -> tuple[AutoRouterBenchmarkGroup, ...]: + """Zeroed groups for configured strategy routers the window's traffic did not cover. + + The dashboard's router picker has to list a router the moment it is created rather than + once it has spent something, so the registry drives the list and the rollup only supplies + the measures. ``_summed_agg_row`` over no sessions is already the zero element of the + fold, so a group with every measure at zero costs one relabel rather than a literal that + would go stale the next time the response grows a field. + """ + if llm_router is None: + return () + zero: Final = _summed_agg_row(()) + idle: Final = frozenset( + key + for key in (_strategy_router_key(deployment) for deployment in llm_router.model_list or ()) + if key is not None and key not in covered + ) + return tuple( + _benchmark_group(zero.model_copy(update=MappingProxyType({"router_name": name, "router_type": kind}))) + for name, kind in sorted(idle) + ) + + @router.get( "/auto_router/benchmarks", tags=("auto router",), @@ -532,8 +580,13 @@ async def get_auto_router_benchmarks( overlaps it: its last turn is on or after start_date and its first turn is on or before end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is over that bucket's turns. + + The rollup supplies the measures, never the list. Which routers appear comes from the + model registry, so one shows up as soon as it is configured and reads zero until it + serves traffic, and `routers_in_scope` counts those too rather than only the routers the + window recorded. """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import llm_router, prisma_client _require_admin_viewer(user_api_key_dict, "view auto-router benchmarks across the deployment") if prisma_client is None: @@ -555,11 +608,14 @@ async def get_auto_router_benchmarks( (end_day + timedelta(days=1)).isoformat(), ) rows: Final = _SESSION_AGG_ROWS.validate_python(raw_rows or ()) - groups: Final = tuple(_benchmark_group(row) for row in rows) + groups: Final = ( + *(_benchmark_group(row) for row in rows), + *_idle_router_groups(llm_router, frozenset((row.router_name, row.router_type) for row in rows)), + ) return AutoRouterBenchmarksResponse( start_date=start_day.strftime("%Y-%m-%d"), end_date=end_day.strftime("%Y-%m-%d"), - routers_in_scope=len(rows), + routers_in_scope=len(groups), totals=_benchmark_totals(_summed_agg_row(rows)), groups=groups, ) diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index e2469d4c78f..a88ffeec6b5 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -158,9 +158,18 @@ class AutoRouterBenchmarksResponse(BaseModel): start_date: str = Field(description="Window start day, YYYY-MM-DD UTC, inclusive") end_date: str = Field(description="Window end day, YYYY-MM-DD UTC, inclusive") - routers_in_scope: int + routers_in_scope: int = Field( + description="How many groups this response carries. Every auto-router configured on the " + "proxy counts, whether or not it served anything in the window. To count only the routers " + "that did serve traffic, filter `groups` to the entries whose `sessions` is above zero" + ) totals: AutoRouterBenchmarkTotals - groups: tuple[AutoRouterBenchmarkGroup, ...] + groups: tuple[AutoRouterBenchmarkGroup, ...] = Field( + description="One entry per auto-router, listed from the model registry rather than from " + "the rollup, so a router appears as soon as it is configured and reads zero until it " + "serves traffic. Semantic auto-routers are absent: they record no routing decision, so no " + "session can ever be attributed to them" + ) ShadowEvalStatus: TypeAlias = Literal["running", "completed", "stopped"] diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index 805168c84ac..3a0279ab0fa 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -2,6 +2,7 @@ Unit tests for auto router management endpoints """ +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Final @@ -22,11 +23,22 @@ from litellm.proxy.management_endpoints.auto_router_endpoints import ( from litellm.router import Router from litellm.types.utils import Choices, Message, ModelResponse from litellm.types.management_endpoints.auto_router_endpoints import ( + AutoRouterBenchmarksResponse, AutoRouterRoutingTestRequest, ) ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test", user_id="admin") + +def _deployment(model_name: str, model: str, *, db_model: bool) -> dict[str, object]: + """One entry as `Router.model_list` holds it, for either origin.""" + return { + "model_name": model_name, + "litellm_params": {"model": model}, + "model_info": {"id": f"{model_name}-{int(db_model)}", "db_model": db_model}, + } + + TIERS = { "SIMPLE": ["cheap-model"], "MEDIUM": ["mid-model"], @@ -295,6 +307,34 @@ def test_classifier_plugin_is_not_settable_over_http(): class TestAutoRouterBenchmarks: from litellm.proxy.management_endpoints.auto_router_endpoints import _SessionAggRow + @pytest.fixture(autouse=True) + def _pin_the_router_global(self, monkeypatch: pytest.MonkeyPatch): + """Every test here reads proxy_server.llm_router, so no test may inherit a sibling's.""" + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "llm_router", None) + + @staticmethod + async def _benchmarks( + monkeypatch: pytest.MonkeyPatch, + rows: Sequence[Mapping[str, object]], + model_list: Sequence[object], + ) -> AutoRouterBenchmarksResponse: + from litellm.proxy import proxy_server + from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks + + class _DB: + async def query_raw(self, sql: str, *params: object): + return rows + + monkeypatch.setattr(proxy_server, "prisma_client", type("P", (), {"db": _DB()})()) + monkeypatch.setattr(proxy_server, "llm_router", type("R", (), {"model_list": model_list})()) + return await get_auto_router_benchmarks( + user_api_key_dict=ADMIN, + start_date="2026-07-01", + end_date="2026-08-01", + ) + ROW = _SessionAggRow( router_name="live-auto", router_type="complexity", @@ -471,6 +511,113 @@ class TestAutoRouterBenchmarks: ) assert response.groups[0].tier_turns == expected + @pytest.mark.asyncio + async def test_the_picker_lists_configured_routers_before_they_have_traffic(self, monkeypatch: pytest.MonkeyPatch): + """A router must be selectable the moment it exists, from either origin. + + `live-auto` is the only router the rollup knows about, so before this it was the only + thing the dropdown could offer. Both a config.yaml router and a DB-created one now + arrive zeroed, and neither moves the totals or duplicates the router that has traffic. + """ + response = await self._benchmarks( + monkeypatch, + rows=[self.ROW.model_dump()], + model_list=[ + _deployment("live-auto", "auto_router/complexity_router", db_model=False), + _deployment("idle-from-config", "auto_router/complexity_router", db_model=False), + _deployment("idle-from-db", "auto_router/complexity_router", db_model=True), + ], + ) + + by_name = {group.router_name: group for group in response.groups} + assert sorted(by_name) == ["idle-from-config", "idle-from-db", "live-auto"] + assert len(response.groups) == 3 + assert response.routers_in_scope == 3 + assert by_name["live-auto"].spend == 10.0 + assert response.totals.spend == 10.0 + assert response.totals.sessions == 4 + for name in ("idle-from-config", "idle-from-db"): + idle = by_name[name] + assert idle.router_type == "complexity" + assert (idle.sessions, idle.turns, idle.spend, idle.saved_spend, idle.baseline_spend) == ( + 0, + 0, + 0.0, + 0.0, + 0.0, + ) + assert (idle.saved_pct, idle.saved_per_session, idle.avg_turns_per_session) == (0.0, 0.0, 0.0) + assert (idle.cache.hit_rate_pct, idle.cache.coverage_pct) == (0.0, 0.0) + assert idle.cache.same_model.turns == idle.cache.return_to_tier.hits == 0 + assert idle.tier_turns == {} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "model, listed_as", + [ + ("auto_router/complexity_router", "complexity"), + ("auto_router/adaptive_router", "adaptive"), + ("auto_router/quality_router", "quality"), + ("auto_router/my-semantic-router", None), + ("openai/gpt-5", None), + ], + ) + async def test_only_kinds_whose_routing_the_rollup_records_are_listed( + self, model: str, listed_as: str | None, monkeypatch: pytest.MonkeyPatch + ): + """A semantic auto-router records no routing decision, so it can never own a session + row; listing it would show $0 forever even while it serves traffic.""" + response = await self._benchmarks( + monkeypatch, rows=[], model_list=[_deployment("candidate", model, db_model=True)] + ) + + assert [group.router_type for group in response.groups] == ([listed_as] if listed_as else []) + + @pytest.mark.asyncio + async def test_a_malformed_deployment_is_skipped_rather_than_failing_the_dashboard( + self, monkeypatch: pytest.MonkeyPatch + ): + response = await self._benchmarks( + monkeypatch, + rows=[self.ROW.model_dump()], + model_list=[ + "not-a-mapping", + {}, + {"model_name": "no-params"}, + {"model_name": "", "litellm_params": {"model": "auto_router/complexity_router"}}, + {"model_name": 7, "litellm_params": {"model": "auto_router/complexity_router"}}, + {"model_name": "no-model", "litellm_params": {}}, + {"model_name": "unreadable-model", "litellm_params": {"model": None}}, + ], + ) + + assert [group.router_name for group in response.groups] == ["live-auto"] + + @pytest.mark.asyncio + async def test_two_deployments_of_one_router_are_listed_once(self, monkeypatch: pytest.MonkeyPatch): + """Tagged variants share a model_name, and the picker selects by name and type.""" + response = await self._benchmarks( + monkeypatch, + rows=[], + model_list=[ + _deployment("tagged", "auto_router/complexity_router", db_model=True), + _deployment("tagged", "auto_router/complexity_router", db_model=True), + ], + ) + + assert [group.router_name for group in response.groups] == ["tagged"] + + def test_the_listed_kinds_match_the_router_types_traffic_can_record(self): + """The one reason semantic is excluded, pinned against both declarations: a kind the + rollup can record must be listable, and a kind it cannot must not be.""" + from typing import get_args, get_type_hints + + from litellm.router_utils.auto_router_model_naming import StrategyRouterKind + from litellm.types.utils import StandardLoggingRoutingDecision + + recorded = set(get_args(get_type_hints(StandardLoggingRoutingDecision)["router_type"])) + assert set(get_args(StrategyRouterKind)) - {"semantic"} == recorded + # --------------------------------------------------------------------------- # Shadow eval endpoints diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 90529eaec48..f4731946492 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -777,6 +777,11 @@ export interface paths { * overlaps it: its last turn is on or after start_date and its first turn is on or before * end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is * over that bucket's turns. + * + * The rollup supplies the measures, never the list. Which routers appear comes from the + * model registry, so one shows up as soon as it is configured and reads zero until it + * serves traffic, and `routers_in_scope` counts those too rather than only the routers the + * window recorded. */ get: operations["get_auto_router_benchmarks_auto_router_benchmarks_get"]; put?: never; @@ -21965,9 +21970,15 @@ export interface components { * @description Window end day, YYYY-MM-DD UTC, inclusive */ end_date: string; - /** Groups */ + /** + * Groups + * @description One entry per auto-router, listed from the model registry rather than from the rollup, so a router appears as soon as it is configured and reads zero until it serves traffic. Semantic auto-routers are absent: they record no routing decision, so no session can ever be attributed to them + */ groups: components["schemas"]["AutoRouterBenchmarkGroup"][]; - /** Routers In Scope */ + /** + * Routers In Scope + * @description How many groups this response carries. Every auto-router configured on the proxy counts, whether or not it served anything in the window. To count only the routers that did serve traffic, filter `groups` to the entries whose `sessions` is above zero + */ routers_in_scope: number; /** * Start Date