diff --git a/litellm/proxy/management_endpoints/auto_router_endpoints.py b/litellm/proxy/management_endpoints/auto_router_endpoints.py index 21e652114bc..415ed1bb76a 100644 --- a/litellm/proxy/management_endpoints/auto_router_endpoints.py +++ b/litellm/proxy/management_endpoints/auto_router_endpoints.py @@ -587,13 +587,10 @@ def _summed_agg_row(rows: Sequence[_SessionAggRow]) -> _SessionAggRow: def _strategy_router_key(deployment: object) -> tuple[str, str] | None: - """``(model_name, kind)`` for a deployment whose routing the session rollup records. + """Return ``(model_name, kind)`` for a configured strategy-router deployment. Kinds come from ``classify_strategy_router_model``, the same rule the Router registers a - deployment by, so this arm cannot disagree with the arm that stamped ``router_type`` onto - the session rows. Semantic auto-routers return None: they record no routing decision, so - they can never own a session row, and ``AutoRouterBenchmarkGroup.router_type`` has no - value for them. A permanent zero would read as "no traffic" rather than "not instrumented". + deployment by, so this arm cannot disagree with the value stamped onto session rows. """ if not isinstance(deployment, Mapping): return None @@ -605,7 +602,7 @@ def _strategy_router_key(deployment: object) -> tuple[str, str] | None: if not isinstance(model, str): return None kind: Final = classify_strategy_router_model(model) - return None if kind is None or kind == "semantic" else (router_name, kind) + return None if kind is None else (router_name, kind) def _idle_router_groups( diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index 6b443026f61..2e6785d0bc5 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -15,7 +15,7 @@ from litellm.litellm_core_utils.internal_call_metadata import ( forwarded_internal_call_metadata, parent_session_kwargs, ) -from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN +from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, StandardLoggingRoutingDecision if TYPE_CHECKING: from semantic_router.routers import SemanticRouter @@ -72,6 +72,7 @@ class AutoRouter(CustomLogger): self.auto_router_config_path: str | None = auto_router_config_path self.auto_router_config: str | None = auto_router_config + self.model_name = model_name self.auto_sync_value = self.DEFAULT_AUTO_SYNC_VALUE self.loaded_routes: list[Route] = self._load_semantic_routing_routes() self.routelayer: SemanticRouter | None = None @@ -179,9 +180,17 @@ class AutoRouter(CustomLogger): message_content: Final = self._extract_text_from_messages(resolved_messages) route_name: Final = await self._matched_route_name(routelayer, message_content, request_kwargs) + routed: Final = route_name or self.default_model return PreRoutingHookResponse( - model=route_name or self.default_model, + model=routed, messages=messages, + routing_decision=StandardLoggingRoutingDecision( + router_model_name=self.model_name, + router_type="semantic", + routed_model=routed, + cause="semantic_route_match" if route_name else "default_fallback", + **({"tier": route_name} if route_name is not None else {}), + ), ) async def _matched_route_name( diff --git a/litellm/types/management_endpoints/auto_router_endpoints.py b/litellm/types/management_endpoints/auto_router_endpoints.py index 88869a1edfb..c608c48eed0 100644 --- a/litellm/types/management_endpoints/auto_router_endpoints.py +++ b/litellm/types/management_endpoints/auto_router_endpoints.py @@ -209,7 +209,7 @@ class AutoRouterBenchmarkGroup(AutoRouterBenchmarkTotals): """One auto-router's slice of the benchmarks.""" router_name: str = Field(description="The auto-router alias requests were sent to") - router_type: str = Field(description="complexity, adaptive or quality") + router_type: str = Field(description="complexity, adaptive, quality or semantic") tier_turns: Mapping[str, int] = Field( default_factory=dict, description="Turns per tier, keyed by the tier name the routing decision recorded at " @@ -236,8 +236,7 @@ class AutoRouterBenchmarksResponse(BaseModel): groups: tuple[AutoRouterBenchmarkGroup, ...] = Field( description="One entry per auto-router, listed from the model registry rather than from " "the rollup, so a router appears as soon as it is configured and reads zero until it " - "serves traffic. Semantic auto-routers are absent: they record no routing decision, so no " - "session can ever be attributed to them" + "serves traffic" ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 44c312e2e77..51d90aea81c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2897,6 +2897,7 @@ RoutingDecisionCause = Literal[ # called. Distinct from "session_affinity_pin", which reports the session_affinity flag pinning # every turn including new asks; this cause only appears when session_affinity is off. "user_turn_continuation", + "semantic_route_match", "default_fallback", "keyword", "quality_tier", @@ -2923,7 +2924,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): """Per-request provenance for a pre-routing strategy (auto-router) decision.""" router_model_name: str - router_type: Literal["complexity", "adaptive", "quality"] + router_type: Literal["complexity", "adaptive", "quality", "semantic"] routed_model: str cause: RoutingDecisionCause tier: str diff --git a/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py b/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py index 2ed4f843711..ae7a0a176ea 100644 --- a/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py +++ b/tests/test_litellm/proxy/db/test_autorouter_session_rollup.py @@ -97,6 +97,20 @@ class TestBuildTransaction: transaction = _build(metadata=_metadata(routing_decision={**ROUTING_DECISION, "tier": "reasoning"})) assert transaction is not None and transaction.tier == "reasoning" + def test_a_semantic_routing_decision_is_carried_onto_the_transaction(self): + transaction = _build( + metadata=_metadata( + routing_decision={ + "router_model_name": "semantic-auto", + "router_type": "semantic", + "routed_model": "code-model", + "cause": "semantic_route_match", + "tier": "code-model", + } + ) + ) + assert transaction is not None and transaction.router_type == "semantic" + @pytest.mark.parametrize("tier", [None, "", 3, {"tier": "medium"}]) def test_a_decision_without_a_usable_tier_records_no_tier(self, tier: object): transaction = _build(metadata=_metadata(routing_decision={**ROUTING_DECISION, "tier": tier})) diff --git a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py index c525af84511..d917ac9b0ac 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_auto_router_endpoints.py @@ -728,21 +728,43 @@ class TestAutoRouterBenchmarks: ("auto_router/complexity_router", "complexity"), ("auto_router/adaptive_router", "adaptive"), ("auto_router/quality_router", "quality"), - ("auto_router/my-semantic-router", None), + ("auto_router/my-semantic-router", "semantic"), ("openai/gpt-5", None), ], ) - async def test_only_kinds_whose_routing_the_rollup_records_are_listed( + async def test_all_strategy_router_kinds_are_listed( self, model: str, listed_as: str | None, monkeypatch: pytest.MonkeyPatch ): - """A semantic auto-router records no routing decision, so it can never own a session - row; listing it would show $0 forever even while it serves traffic.""" + """Every strategy auto-router kind is listed before it has traffic.""" response = await self._benchmarks( monkeypatch, rows=[], model_list=[_deployment("candidate", model, db_model=True)] ) assert [group.router_type for group in response.groups] == ([listed_as] if listed_as else []) + @pytest.mark.asyncio + async def test_a_semantic_deployment_is_listed_before_it_has_traffic(self, monkeypatch: pytest.MonkeyPatch): + from litellm.proxy.management_endpoints.auto_router_endpoints import _strategy_router_key + + deployment = { + "model_name": "semantic-router", + "litellm_params": { + "model": "auto_router/auto_router_1", + "auto_router_config_path": "routes.json", + "auto_router_default_model": "fallback-model", + "auto_router_embedding_model": "embedding-model", + }, + "model_info": {"id": "semantic-router-1", "db_model": True}, + } + + assert _strategy_router_key(deployment) == ("semantic-router", "semantic") + response = await self._benchmarks(monkeypatch, rows=[], model_list=[deployment]) + + assert len(response.groups) == 1 + assert response.groups[0].router_name == "semantic-router" + assert response.groups[0].router_type == "semantic" + assert response.groups[0].turns == 0 + @pytest.mark.asyncio async def test_a_malformed_deployment_is_skipped_rather_than_failing_the_dashboard( self, monkeypatch: pytest.MonkeyPatch @@ -778,15 +800,14 @@ class TestAutoRouterBenchmarks: assert [group.router_name for group in response.groups] == ["tagged"] def test_the_listed_kinds_match_the_router_types_traffic_can_record(self): - """The one reason semantic is excluded, pinned against both declarations: a kind the - rollup can record must be listable, and a kind it cannot must not be.""" + """Every strategy-router kind can be listed and recorded by the rollup.""" from typing import get_args, get_type_hints from litellm.router_utils.auto_router_model_naming import StrategyRouterKind from litellm.types.utils import StandardLoggingRoutingDecision recorded = set(get_args(get_type_hints(StandardLoggingRoutingDecision)["router_type"])) - assert set(get_args(StrategyRouterKind)) - {"semantic"} == recorded + assert set(get_args(StrategyRouterKind)) == recorded # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/router_strategy/test_auto_router.py b/tests/test_litellm/router_strategy/test_auto_router.py index 123ada83ca4..392425da02c 100644 --- a/tests/test_litellm/router_strategy/test_auto_router.py +++ b/tests/test_litellm/router_strategy/test_auto_router.py @@ -415,6 +415,12 @@ class TestAutoRouterAlwaysResolvesARoutableModel: assert result is not None # Leaving "my-auto-router" here fails downstream with "Unmapped LLM provider". assert result.model == "fallback-model" + assert result.routing_decision == { + "router_type": "semantic", + "router_model_name": "my-auto-router", + "routed_model": "fallback-model", + "cause": "default_fallback", + } @pytest.mark.asyncio async def test_should_fall_back_to_default_model_when_the_route_layer_returns_an_empty_list(self): @@ -461,6 +467,13 @@ class TestAutoRouterAlwaysResolvesARoutableModel: assert result is not None assert result.model == "code-model" + assert result.routing_decision == { + "router_type": "semantic", + "router_model_name": "my-auto-router", + "routed_model": "code-model", + "cause": "semantic_route_match", + "tier": "code-model", + } assert router.seen_text == "fix this stack trace" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index d94b5425ba9..19a56a681b5 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -23276,7 +23276,7 @@ export interface components { router_name: string; /** * Router Type - * @description complexity, adaptive or quality + * @description complexity, adaptive, quality or semantic */ router_type: string; /** @@ -23359,7 +23359,7 @@ export interface components { end_date: string; /** * Groups - * @description One entry per auto-router, listed from the model registry rather than from the rollup, so a router appears as soon as it is configured and reads zero until it serves traffic. Semantic auto-routers are absent: they record no routing decision, so no session can ever be attributed to them + * @description One entry per auto-router, listed from the model registry rather than from the rollup, so a router appears as soon as it is configured and reads zero until it serves traffic */ groups: components["schemas"]["AutoRouterBenchmarkGroup"][]; /** @@ -36045,7 +36045,7 @@ export interface components { * Cause * @enum {string} */ - cause?: "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "modality_pin_override" | "health_failover" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; + cause?: "heuristic_scorer" | "heuristic_v2" | "reasoning_override" | "llm_classifier" | "heuristic_first_short_circuit" | "hybrid_short_circuit" | "classifier_plugin" | "classifier_fallback" | "default_model_fallback" | "literal_keyword_match" | "semantic_keyword_match" | "plan_mode" | "housekeeping" | "modality_escalation" | "modality_pin_override" | "health_failover" | "session_affinity_pin" | "session_affinity_escalation" | "user_turn_continuation" | "semantic_route_match" | "default_fallback" | "keyword" | "quality_tier" | "bandit"; /** Classifier Cost */ classifier_cost?: number; /** Classifier Model */ @@ -36074,7 +36074,7 @@ export interface components { * Router Type * @enum {string} */ - router_type?: "complexity" | "adaptive" | "quality"; + router_type?: "complexity" | "adaptive" | "quality" | "semantic"; /** Savings Baseline Deployment Id */ savings_baseline_deployment_id?: string; /** Savings Baseline Model */