scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

fix(auto-router): list configured auto-routers in the usage picker before they have traffic

Browse source
This commit is contained in:
Tin Chi Lo 2026-08-24 15:28:20 -04:00
parent b36f34813a
commit 26e47aea32
4 changed files with 230 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
litellm/proxy/management_endpoints/auto_router_endpoints.py
View file

@ -36,6 +36,7 @@ from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.repositories.base_repository import SupportsModelDump
from litellm.repositories.team_repository import TeamRepository
from litellm.router_strategy.complexity_router import ComplexityRouter
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model
from litellm.types.management_endpoints.auto_router_endpoints import (
SHADOW_EVAL_TURN_VALVE,
AutoRouterBenchmarkGroup,
@ -510,6 +511,53 @@ def _summed_agg_row(rows: Sequence[_SessionAggRow]) -> _SessionAggRow:
)
def _strategy_router_key(deployment: object) -> tuple[str, str] | None:
"""``(model_name, kind)`` for a deployment whose routing the session rollup records.
Kinds come from ``classify_strategy_router_model``, the same rule the Router registers a
deployment by, so this arm cannot disagree with the arm that stamped ``router_type`` onto
the session rows. Semantic auto-routers return None: they record no routing decision, so
they can never own a session row, and ``AutoRouterBenchmarkGroup.router_type`` has no
value for them. A permanent zero would read as "no traffic" rather than "not instrumented".
"""
if not isinstance(deployment, Mapping):
return None
litellm_params: Final = deployment.get("litellm_params")
router_name: Final = deployment.get("model_name")
if not (isinstance(litellm_params, Mapping) and isinstance(router_name, str) and router_name):
return None
model: Final = litellm_params.get("model")
if not isinstance(model, str):
return None
kind: Final = classify_strategy_router_model(model)
return None if kind is None or kind == "semantic" else (router_name, kind)
def _idle_router_groups(
llm_router: "Router | None", covered: frozenset[tuple[str, str]]
) -> tuple[AutoRouterBenchmarkGroup, ...]:
"""Zeroed groups for configured strategy routers the window's traffic did not cover.
The dashboard's router picker has to list a router the moment it is created rather than
once it has spent something, so the registry drives the list and the rollup only supplies
the measures. ``_summed_agg_row`` over no sessions is already the zero element of the
fold, so a group with every measure at zero costs one relabel rather than a literal that
would go stale the next time the response grows a field.
"""
if llm_router is None:
return ()
zero: Final = _summed_agg_row(())
idle: Final = frozenset(
key
for key in (_strategy_router_key(deployment) for deployment in llm_router.model_list or ())
if key is not None and key not in covered
)
return tuple(
_benchmark_group(zero.model_copy(update=MappingProxyType({"router_name": name, "router_type": kind})))
for name, kind in sorted(idle)
)
@router.get(
"/auto_router/benchmarks",
tags=("auto router",),
@ -532,8 +580,13 @@ async def get_auto_router_benchmarks(
overlaps it: its last turn is on or after start_date and its first turn is on or before
end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is
over that bucket's turns.
The rollup supplies the measures, never the list. Which routers appear comes from the
model registry, so one shows up as soon as it is configured and reads zero until it
serves traffic, and `routers_in_scope` counts those too rather than only the routers the
window recorded.
"""
from litellm.proxy.proxy_server import prisma_client
from litellm.proxy.proxy_server import llm_router, prisma_client
_require_admin_viewer(user_api_key_dict, "view auto-router benchmarks across the deployment")
if prisma_client is None:
@ -555,11 +608,14 @@ async def get_auto_router_benchmarks(
(end_day + timedelta(days=1)).isoformat(),
)
rows: Final = _SESSION_AGG_ROWS.validate_python(raw_rows or ())
groups: Final = tuple(_benchmark_group(row) for row in rows)
groups: Final = (
*(_benchmark_group(row) for row in rows),
*_idle_router_groups(llm_router, frozenset((row.router_name, row.router_type) for row in rows)),
)
return AutoRouterBenchmarksResponse(
start_date=start_day.strftime("%Y-%m-%d"),
end_date=end_day.strftime("%Y-%m-%d"),
routers_in_scope=len(rows),
routers_in_scope=len(groups),
totals=_benchmark_totals(_summed_agg_row(rows)),
groups=groups,
)

13
litellm/types/management_endpoints/auto_router_endpoints.py
View file

@ -158,9 +158,18 @@ class AutoRouterBenchmarksResponse(BaseModel):
start_date: str = Field(description="Window start day, YYYY-MM-DD UTC, inclusive")
end_date: str = Field(description="Window end day, YYYY-MM-DD UTC, inclusive")
routers_in_scope: int
routers_in_scope: int = Field(
description="How many groups this response carries. Every auto-router configured on the "
"proxy counts, whether or not it served anything in the window. To count only the routers "
"that did serve traffic, filter `groups` to the entries whose `sessions` is above zero"
)
totals: AutoRouterBenchmarkTotals
groups: tuple[AutoRouterBenchmarkGroup, ...]
groups: tuple[AutoRouterBenchmarkGroup, ...] = Field(
description="One entry per auto-router, listed from the model registry rather than from "
"the rollup, so a router appears as soon as it is configured and reads zero until it "
"serves traffic. Semantic auto-routers are absent: they record no routing decision, so no "
"session can ever be attributed to them"
)
ShadowEvalStatus: TypeAlias = Literal["running", "completed", "stopped"]

147
tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py
View file

@ -2,6 +2,7 @@
Unit tests for auto router management endpoints
"""
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Final
@ -22,11 +23,22 @@ from litellm.proxy.management_endpoints.auto_router_endpoints import (
from litellm.router import Router
from litellm.types.utils import Choices, Message, ModelResponse
from litellm.types.management_endpoints.auto_router_endpoints import (
AutoRouterBenchmarksResponse,
AutoRouterRoutingTestRequest,
)
ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="sk-test", user_id="admin")
def _deployment(model_name: str, model: str, *, db_model: bool) -> dict[str, object]:
"""One entry as `Router.model_list` holds it, for either origin."""
return {
"model_name": model_name,
"litellm_params": {"model": model},
"model_info": {"id": f"{model_name}-{int(db_model)}", "db_model": db_model},
}
TIERS = {
"SIMPLE": ["cheap-model"],
"MEDIUM": ["mid-model"],
@ -295,6 +307,34 @@ def test_classifier_plugin_is_not_settable_over_http():
class TestAutoRouterBenchmarks:
from litellm.proxy.management_endpoints.auto_router_endpoints import _SessionAggRow
@pytest.fixture(autouse=True)
def _pin_the_router_global(self, monkeypatch: pytest.MonkeyPatch):
"""Every test here reads proxy_server.llm_router, so no test may inherit a sibling's."""
from litellm.proxy import proxy_server
monkeypatch.setattr(proxy_server, "llm_router", None)
@staticmethod
async def _benchmarks(
monkeypatch: pytest.MonkeyPatch,
rows: Sequence[Mapping[str, object]],
model_list: Sequence[object],
) -> AutoRouterBenchmarksResponse:
from litellm.proxy import proxy_server
from litellm.proxy.management_endpoints.auto_router_endpoints import get_auto_router_benchmarks
class _DB:
async def query_raw(self, sql: str, *params: object):
return rows
monkeypatch.setattr(proxy_server, "prisma_client", type("P", (), {"db": _DB()})())
monkeypatch.setattr(proxy_server, "llm_router", type("R", (), {"model_list": model_list})())
return await get_auto_router_benchmarks(
user_api_key_dict=ADMIN,
start_date="2026-07-01",
end_date="2026-08-01",
)
ROW = _SessionAggRow(
router_name="live-auto",
router_type="complexity",
@ -471,6 +511,113 @@ class TestAutoRouterBenchmarks:
)
assert response.groups[0].tier_turns == expected
@pytest.mark.asyncio
async def test_the_picker_lists_configured_routers_before_they_have_traffic(self, monkeypatch: pytest.MonkeyPatch):
"""A router must be selectable the moment it exists, from either origin.
`live-auto` is the only router the rollup knows about, so before this it was the only
thing the dropdown could offer. Both a config.yaml router and a DB-created one now
arrive zeroed, and neither moves the totals or duplicates the router that has traffic.
"""
response = await self._benchmarks(
monkeypatch,
rows=[self.ROW.model_dump()],
model_list=[
_deployment("live-auto", "auto_router/complexity_router", db_model=False),
_deployment("idle-from-config", "auto_router/complexity_router", db_model=False),
_deployment("idle-from-db", "auto_router/complexity_router", db_model=True),
],
)
by_name = {group.router_name: group for group in response.groups}
assert sorted(by_name) == ["idle-from-config", "idle-from-db", "live-auto"]
assert len(response.groups) == 3
assert response.routers_in_scope == 3
assert by_name["live-auto"].spend == 10.0
assert response.totals.spend == 10.0
assert response.totals.sessions == 4
for name in ("idle-from-config", "idle-from-db"):
idle = by_name[name]
assert idle.router_type == "complexity"
assert (idle.sessions, idle.turns, idle.spend, idle.saved_spend, idle.baseline_spend) == (
0,
0,
0.0,
0.0,
0.0,
)
assert (idle.saved_pct, idle.saved_per_session, idle.avg_turns_per_session) == (0.0, 0.0, 0.0)
assert (idle.cache.hit_rate_pct, idle.cache.coverage_pct) == (0.0, 0.0)
assert idle.cache.same_model.turns == idle.cache.return_to_tier.hits == 0
assert idle.tier_turns == {}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model, listed_as",
[
("auto_router/complexity_router", "complexity"),
("auto_router/adaptive_router", "adaptive"),
("auto_router/quality_router", "quality"),
("auto_router/my-semantic-router", None),
("openai/gpt-5", None),
],
)
async def test_only_kinds_whose_routing_the_rollup_records_are_listed(
self, model: str, listed_as: str | None, monkeypatch: pytest.MonkeyPatch
):
"""A semantic auto-router records no routing decision, so it can never own a session
row; listing it would show $0 forever even while it serves traffic."""
response = await self._benchmarks(
monkeypatch, rows=[], model_list=[_deployment("candidate", model, db_model=True)]
)
assert [group.router_type for group in response.groups] == ([listed_as] if listed_as else [])
@pytest.mark.asyncio
async def test_a_malformed_deployment_is_skipped_rather_than_failing_the_dashboard(
self, monkeypatch: pytest.MonkeyPatch
):
response = await self._benchmarks(
monkeypatch,
rows=[self.ROW.model_dump()],
model_list=[
"not-a-mapping",
{},
{"model_name": "no-params"},
{"model_name": "", "litellm_params": {"model": "auto_router/complexity_router"}},
{"model_name": 7, "litellm_params": {"model": "auto_router/complexity_router"}},
{"model_name": "no-model", "litellm_params": {}},
{"model_name": "unreadable-model", "litellm_params": {"model": None}},
],
)
assert [group.router_name for group in response.groups] == ["live-auto"]
@pytest.mark.asyncio
async def test_two_deployments_of_one_router_are_listed_once(self, monkeypatch: pytest.MonkeyPatch):
"""Tagged variants share a model_name, and the picker selects by name and type."""
response = await self._benchmarks(
monkeypatch,
rows=[],
model_list=[
_deployment("tagged", "auto_router/complexity_router", db_model=True),
_deployment("tagged", "auto_router/complexity_router", db_model=True),
],
)
assert [group.router_name for group in response.groups] == ["tagged"]
def test_the_listed_kinds_match_the_router_types_traffic_can_record(self):
"""The one reason semantic is excluded, pinned against both declarations: a kind the
rollup can record must be listable, and a kind it cannot must not be."""
from typing import get_args, get_type_hints
from litellm.router_utils.auto_router_model_naming import StrategyRouterKind
from litellm.types.utils import StandardLoggingRoutingDecision
recorded = set(get_args(get_type_hints(StandardLoggingRoutingDecision)["router_type"]))
assert set(get_args(StrategyRouterKind)) - {"semantic"} == recorded
# ---------------------------------------------------------------------------
# Shadow eval endpoints

15
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -777,6 +777,11 @@ export interface paths {
* overlaps it: its last turn is on or after start_date and its first turn is on or before
* end_date. Overall hit rate is over telemetry-bearing turns; each bucket's hit rate is
* over that bucket's turns.
*
* The rollup supplies the measures, never the list. Which routers appear comes from the
* model registry, so one shows up as soon as it is configured and reads zero until it
* serves traffic, and `routers_in_scope` counts those too rather than only the routers the
* window recorded.
*/
get: operations["get_auto_router_benchmarks_auto_router_benchmarks_get"];
put?: never;
@ -21965,9 +21970,15 @@ export interface components {
* @description Window end day, YYYY-MM-DD UTC, inclusive
*/
end_date: string;
/** Groups */
/**
* Groups
* @description One entry per auto-router, listed from the model registry rather than from the rollup, so a router appears as soon as it is configured and reads zero until it serves traffic. Semantic auto-routers are absent: they record no routing decision, so no session can ever be attributed to them
*/
groups: components["schemas"]["AutoRouterBenchmarkGroup"][];
/** Routers In Scope */
/**
* Routers In Scope
* @description How many groups this response carries. Every auto-router configured on the proxy counts, whether or not it served anything in the window. To count only the routers that did serve traffic, filter `groups` to the entries whose `sessions` is above zero
*/
routers_in_scope: number;
/**
* Start Date