feat(auto-router): track turns per complexity tier (LIT-5302)

Stamps `tier` onto the routing_decision at the call sites that lacked it
(session-affinity pin/escalation, default_fallback-via-MEDIUM), then rolls
a per-tier turn counter into LiteLLM_AutoRouterSession.tier_turns (jsonb,
keyed by tier name). The benchmarks endpoint aggregates it per-router via
jsonb_object_agg and returns it on AutoRouterBenchmarkGroup.tier_turns.

Tier is recorded once, at decision time, and summed at read time — never
re-derived from the routed model, since the tier->model mapping is mutable
config and a 30-day window would otherwise report today's config against
turns routed under yesterday's.

jsonb keyed by tier name (not fixed columns) absorbs PR #35915's
operator-defined tier_definitions without a migration per taxonomy.

No API surface change beyond the new tier_turns field: GET /auto_router/benchmarks
stays admin-gated, unchanged.

litellm-proxy-extras/litellm_proxy_extras/migrations/20260807000000_add_autorouter_session_tier_turns/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_AutoRouterSession" ADD COLUMN IF NOT EXISTS "tier_turns" JSONB NOT NULL DEFAULT '{}';

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1439,6 +1439,7 @@ model LiteLLM_AutoRouterSession {
   total_tokens          BigInt   @default(0)
   spend                 Float    @default(0)
   saved_spend           Float    @default(0)
+  tier_turns            Json     @default("{}")
 
   @@id([api_key, session_id, router_name])
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")

litellm/proxy/db/autorouter_session_rollup.py

@@ -49,6 +49,11 @@ class AutoRouterTurnTransaction:
     cache_hit: bool
     cache_ttl_seconds: int | None
     cache_touched: bool
+    # The complexity tier the router decided (or resolved a pinned model to) for this
+    # turn, exactly as the routing_decision recorded it. None when no tier served the
+    # turn (classifier fell back to default_model, or a non-complexity router), and
+    # such turns increment no tier counter rather than a synthetic "unknown" bucket.
+    tier: str | None = None
 
 
 class TurnCacheFacts(NamedTuple):
@@ -152,11 +157,13 @@ def build_autorouter_turn_transaction(
         return None
     usage_object_raw: Final = metadata.get("usage_object")
     cache: Final = turn_cache_facts(usage_object_raw if isinstance(usage_object_raw, Mapping) else None)
+    tier_raw: Final = routing_decision.get("tier")
     return AutoRouterTurnTransaction(
         api_key=api_key,
         session_id=_bounded_session_id(session_id),
         router_name=router_name,
         router_type=str(routing_decision.get("router_type") or "unknown"),
+        tier=tier_raw if isinstance(tier_raw, str) and tier_raw else None,
         model=model,
         turn_at=turn_at,
         total_tokens=int(payload.get("prompt_tokens") or 0) + int(payload.get("completion_tokens") or 0),
@@ -184,6 +191,14 @@ _COVERED: Final = _p("covered")
 _CACHE_HIT: Final = _p("cache_hit")
 _CACHE_TTL: Final = _p("cache_ttl_seconds")
 _TOUCHED: Final = _p("cache_touched")
+_TIER: Final = _p("tier")
+
+# One key per tier that served a turn, incremented per turn. A NULL tier increments
+# nothing: '{}' both on insert and on update, so untiered turns are simply absent
+# rather than pooled under a sentinel key the dashboard would have to special-case.
+_TIER_TURNS_DELTA: Final = (
+    f"(CASE WHEN {_TIER} IS NOT NULL THEN jsonb_build_object({_TIER}, 1) ELSE '{{}}'::jsonb END)"
+)
 
 _IN_ORDER: Final = f"{_TURN_AT}::timestamp >= t.last_turn_at"
 _SAME: Final = f"{_IN_ORDER} AND t.last_model = {_MODEL}"
@@ -201,7 +216,7 @@ INSERT INTO "LiteLLM_AutoRouterSession" AS t (
     last_model, models, turns, unordered_turns, covered_turns, cache_hits,
     same_model_turns, same_model_hits, first_visit_turns, first_visit_hits,
     return_turns, return_hits, return_expired_misses, return_within_ttl_misses,
-    ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend
+    ttl_5m_turns, ttl_1h_turns, total_tokens, spend, saved_spend, tier_turns
 )
 VALUES (
     {_p("api_key")}, {_p("session_id")}, {_p("router_name")}, {_p("router_type")}, {_TURN_AT}::timestamp, {_TURN_AT}::timestamp,
@@ -211,7 +226,8 @@ VALUES (
     0, 0, 0, 0,
     (CASE WHEN {_CACHE_TTL}::int = {CACHE_TTL_5M_SECONDS} THEN 1 ELSE 0 END),
     (CASE WHEN {_CACHE_TTL}::int = {CACHE_TTL_1H_SECONDS} THEN 1 ELSE 0 END),
-    {_p("total_tokens")}::bigint, {_p("spend")}::float8, {_p("saved_spend")}::float8
+    {_p("total_tokens")}::bigint, {_p("spend")}::float8, {_p("saved_spend")}::float8,
+    {_TIER_TURNS_DELTA}
 )
 ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
     turns = t.turns + 1,
@@ -242,6 +258,9 @@ ON CONFLICT (api_key, session_id, router_name) DO UPDATE SET
                 ELSE COALESCE((t.models -> {_MODEL} ->> 'ttl')::int, {_CACHE_TTL}::int) END)
     )),
     last_model = (CASE WHEN {_IN_ORDER} THEN {_MODEL} ELSE t.last_model END),
+    tier_turns = (CASE WHEN {_TIER} IS NOT NULL
+        THEN t.tier_turns || jsonb_build_object({_TIER}, COALESCE((t.tier_turns ->> {_TIER})::int, 0) + 1)
+        ELSE t.tier_turns END),
     first_turn_at = LEAST(t.first_turn_at, EXCLUDED.first_turn_at),
     last_turn_at = GREATEST(t.last_turn_at, EXCLUDED.last_turn_at)
 """

litellm/proxy/management_endpoints/auto_router_endpoints.py

@@ -4,11 +4,12 @@ AUTO ROUTER MANAGEMENT ENDPOINTS
 POST /auto_router/test_routing - Route one prompt through an unsaved complexity-router config
 """
 
+import json
 from collections.abc import Sequence
 from datetime import datetime, timedelta, timezone
 from typing import TYPE_CHECKING, Annotated, Final
 
-from pydantic import BaseModel, TypeAdapter
+from pydantic import BaseModel, Field, TypeAdapter, field_validator
 
 from litellm._logging import verbose_proxy_logger
 from litellm.exceptions import BudgetExceededError
@@ -284,6 +285,23 @@ class _SessionAggRow(BaseModel):
 _SESSION_AGG_ROWS: Final = TypeAdapter(list[_SessionAggRow])
 
 _BENCHMARKS_SQL: Final = """
+WITH windowed AS (
+    SELECT * FROM "LiteLLM_AutoRouterSession"
+    WHERE last_turn_at >= $1::timestamp AND first_turn_at < $2::timestamp
+),
+tier_maps AS (
+    SELECT router_name, router_type, jsonb_object_agg(tier, tier_turns) AS tier_turns
+    FROM (
+        SELECT router_name, router_type, kv.key AS tier, SUM((kv.value)::int)::int AS tier_turns
+        FROM windowed, LATERAL jsonb_each_text(tier_turns) AS kv
+        GROUP BY router_name, router_type, kv.key
+    ) per_tier
+    GROUP BY router_name, router_type
+)
+SELECT
+    agg.*,
+    COALESCE(tier_maps.tier_turns, '{}'::jsonb)::text AS tier_turns
+FROM (
 SELECT
     router_name,
     router_type,
@@ -306,10 +324,11 @@ SELECT
     COALESCE(SUM(spend), 0)::float8 AS spend,
     COALESCE(SUM(saved_spend), 0)::float8 AS saved_spend,
     COALESCE(SUM(EXTRACT(EPOCH FROM (last_turn_at - first_turn_at))), 0)::float8 AS session_seconds
-FROM "LiteLLM_AutoRouterSession"
-WHERE last_turn_at >= $1::timestamp AND first_turn_at < $2::timestamp
+FROM windowed
 GROUP BY router_name, router_type
-ORDER BY SUM(spend) DESC
+) agg
+LEFT JOIN tier_maps USING (router_name, router_type)
+ORDER BY agg.spend DESC
 """
 
 
@@ -443,6 +462,7 @@ async def get_auto_router_benchmarks(
         AutoRouterBenchmarkGroup(
             router_name=row.router_name,
             router_type=row.router_type,
+            tier_turns=row.tier_turns,
             **_benchmark_totals(row).model_dump(),
         )
         for row in rows

litellm/proxy/schema.prisma

@@ -1439,6 +1439,7 @@ model LiteLLM_AutoRouterSession {
   total_tokens          BigInt   @default(0)
   spend                 Float    @default(0)
   saved_spend           Float    @default(0)
+  tier_turns            Json     @default("{}")
 
   @@id([api_key, session_id, router_name])
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")

litellm/router_strategy/complexity_router/complexity_router.py

@@ -1730,6 +1730,13 @@ class ComplexityRouter(CustomLogger):
                         routing_decision=self._build_routing_decision(
                             routed_model=routed_model,
                             cause=cause,
+                            # No classification ran, but the model this session is pinned to still
+                            # belongs to a tier, and that tier is what actually served the turn. It
+                            # is resolved against the config live at this turn (not the one that set
+                            # the pin) so the record matches the pools the request was served from.
+                            # None when the pin no longer maps to any configured tier, i.e. the
+                            # operator removed the model from every pool while the session was live.
+                            tier=self._tier_for_model(routed_model),
                             escalation_keyword=pin_escalation_keyword,
                             escalated=escalated,
                             conversation_continuing=conversation_continuing,
@@ -1797,6 +1804,7 @@ class ComplexityRouter(CustomLogger):
 
         if user_message is None:
             verbose_router_logger.debug("ComplexityRouter: No user message found, routing to default model")
+            fallback_tier: ComplexityTier | None = None
             if not self.config.plugins and self.config.default_model:
                 # No plugins configured: preserve the pre-existing default_model-first
                 # priority exactly (changing it would be a silent behavior change for
@@ -1809,12 +1817,22 @@ class ComplexityRouter(CustomLogger):
                 routed_model = await self._pick_model_for_tier(
                     ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs
                 )
+                # This branch asked the MEDIUM pool for the model, so MEDIUM is the tier that
+                # served the turn -- but only when a MEDIUM pool is actually configured.
+                # get_model_for_tier's own branch order (tier pool, then default_model) falls
+                # back to default_model when MEDIUM has no pool, and that fallback is not a
+                # MEDIUM-tier serve; mirroring the same condition here (rather than resolving
+                # the returned model back through _tier_for_model) avoids miscounting a
+                # default_model that happens to also sit in some other tier's pool.
+                if ComplexityTier.MEDIUM.value in self.config.tiers:
+                    fallback_tier = ComplexityTier.MEDIUM
             return PreRoutingHookResponse(
                 model=routed_model,
                 messages=messages if has_original_messages else None,
                 routing_decision=self._build_routing_decision(
                     routed_model=routed_model,
                     cause="default_fallback",
+                    tier=fallback_tier,
                     conversation_continuing=conversation_continuing,
                 ),
             )

litellm/types/management_endpoints/auto_router_endpoints.py

@@ -120,6 +120,14 @@ class AutoRouterBenchmarkGroup(AutoRouterBenchmarkTotals):
 
     router_name: str = Field(description="The auto-router alias requests were sent to")
     router_type: str = Field(description="complexity, adaptive or quality")
+    tier_turns: dict[str, int] = Field(
+        default_factory=dict,
+        description="Turns per complexity tier, keyed by the tier name the routing decision "
+        "recorded at request time (never re-derived from the routed model). Only complexity "
+        "routers produce tiers; turns that no tier served (classifier fell back to "
+        "default_model) are absent, so the values may sum to less than turns. Per-group only: "
+        "tier names are router-scoped, so a cross-router union would mix taxonomies",
+    )
 
 
 class AutoRouterBenchmarksResponse(BaseModel):

schema.prisma

@@ -1439,6 +1439,7 @@ model LiteLLM_AutoRouterSession {
   total_tokens          BigInt   @default(0)
   spend                 Float    @default(0)
   saved_spend           Float    @default(0)
+  tier_turns            Json     @default("{}")
 
   @@id([api_key, session_id, router_name])
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")