From 41479d07f3fe18a52df6b9f72afa7a5bece32722 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Tue, 4 Aug 2026 13:24:02 -0700 Subject: [PATCH] fix(spend): record the router kind the router chose instead of deriving it again Which strategy served a request was being re-derived in the spend writer by checking the router's registries in a fixed order. One alias can own entries in several of them, and which one runs depends on the request's tags, which only Router._select_pre_routing_strategy resolves. The re-derivation therefore disagreed with what actually ran on exactly the tagged deployments that disambiguation exists for, and a third derivation on the read path labelled the dashboard card from config rather than from the traffic underneath it. The router already records its choice on the routing decision, so the writer now reads router_type off that and the read path reads the router_kind stored on the rows. Semantic auto-routers were the one strategy that never emitted a decision, which is why reading it was not previously an option; AutoRouter now emits one like the other three, and router_type gained the "semantic" member it needs. What remains in the writer is a membership test rather than a classifier: does any pre-routing strategy sit under this group at all. That question has one answer regardless of tags, costs four dict lookups, and keeps non-auto-routed traffic from paying to deserialize a metadata blob it will not use. --- litellm/proxy/db/db_spend_update_writer.py | 14 ++++-- .../spend_tracking/auto_router_benchmarks.py | 17 ++++--- .../spend_tracking/auto_router_sessions.py | 47 ++++++++----------- .../auto_router/auto_router.py | 14 +++++- litellm/types/utils.py | 2 +- .../test_auto_router_benchmarks.py | 8 ++-- .../test_auto_router_sessions.py | 38 +++++++-------- 7 files changed, 75 insertions(+), 65 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index bcc90b9ebd5..23e8a132f94 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -59,7 +59,7 @@ from litellm.proxy.db.db_transaction_queue.tool_discovery_queue import ( from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING from litellm.proxy.spend_tracking.auto_router_session_queue import AutoRouterSessionQueue from litellm.proxy.spend_tracking.auto_router_sessions import ( - auto_router_kind, + serves_an_auto_router, turn_from_spend_payload, ) from litellm.proxy.spend_tracking.compression_savings import ( @@ -305,8 +305,7 @@ class DBSpendUpdateWriter: model = payload.get("model") if prisma_client is None or llm_router is None or not session_id or not model_group or not model: return - router_kind = auto_router_kind(llm_router, model_group) - if router_kind is None: + if not serves_an_auto_router(llm_router, model_group): return start_time = payload.get("startTime") started_at = start_time if isinstance(start_time, datetime) else _parse_start_time(start_time) @@ -314,6 +313,13 @@ class DBSpendUpdateWriter: return _metadata: SpendLogsMetadata = json.loads(payload["metadata"]) + routing_decision = _metadata.get("routing_decision") + # The kind is read off the decision the router recorded rather than derived + # again here: one alias can own several strategies and which of them ran + # depends on the request's tags, which only the router resolved. + router_kind = routing_decision.get("router_type") if routing_decision else None + if router_kind is None: + return usage_obj = _metadata.get("usage_object", {}) or {} # mutable-ok: empty fallback for an absent usage payload cache_read_tokens = _extract_cache_read_tokens(usage_obj) savings_spend = compute_savings_spend( @@ -321,7 +327,7 @@ class DBSpendUpdateWriter: custom_llm_provider=payload.get("custom_llm_provider", None), compression_saved_tokens=extract_compression_saved_tokens(_metadata), cache_read_input_tokens=cache_read_tokens, - routing_decision=_metadata.get("routing_decision"), + routing_decision=routing_decision, model_id=payload.get("model_id"), llm_router=_get_llm_router, usage_object=usage_obj, diff --git a/litellm/proxy/spend_tracking/auto_router_benchmarks.py b/litellm/proxy/spend_tracking/auto_router_benchmarks.py index a9f298d2ee4..9838ba33300 100644 --- a/litellm/proxy/spend_tracking/auto_router_benchmarks.py +++ b/litellm/proxy/spend_tracking/auto_router_benchmarks.py @@ -98,6 +98,7 @@ class _GroupRow(BaseModel): model_group: str baseline_model: str | None + router_kind: str sessions: int turns: int total_session_seconds: float @@ -131,6 +132,7 @@ _GROUP_SQL = """ SELECT model_group, MAX(baseline_model) AS baseline_model, + MAX(router_kind) AS router_kind, COUNT(*)::bigint AS sessions, COALESCE(SUM(turns), 0)::bigint AS turns, COALESCE(SUM(EXTRACT(EPOCH FROM (last_turn_at - first_turn_at))), 0) AS total_session_seconds, @@ -226,9 +228,15 @@ def summarize_cache(row: _GroupRow) -> AutoRouterCacheBenchmark | None: ) -def summarize_group(row: _GroupRow, router_kind: str) -> AutoRouterGroupBenchmark | None: +def summarize_group(row: _GroupRow) -> AutoRouterGroupBenchmark | None: """Fold one group's session rows into its benchmark. + ``router_kind`` comes off the rows rather than from the configured groups, + because the rows recorded which strategy actually served each turn while the + config only says which are registered under the alias. Those differ when one + alias owns several tagged strategies, and labelling the card from the config + would name a router the numbers underneath it did not come from. + ``savings`` keeps its sign. A router that thrashes the prompt cache can cost more than the baseline it is measured against, and an operator needs to be able to see that rather than have it floored to zero. @@ -238,7 +246,7 @@ def summarize_group(row: _GroupRow, router_kind: str) -> AutoRouterGroupBenchmar savings = row.baseline_spend - row.actual_spend return AutoRouterGroupBenchmark( model_group=row.model_group, - router_kind=router_kind, + router_kind=row.router_kind, baseline_model=row.baseline_model, sessions=row.sessions, turns=row.turns, @@ -268,10 +276,7 @@ async def compute_benchmarks( window.start, window.end, ) - summarized = ( - summarize_group(row, group_kinds.get(row.model_group, "auto_router")) - for row in _GROUP_ROWS.validate_python(raw) - ) + summarized = (summarize_group(row) for row in _GROUP_ROWS.validate_python(raw)) return AutoRouterBenchmarksResponse( start_date=window.start, end_date=window.end, diff --git a/litellm/proxy/spend_tracking/auto_router_sessions.py b/litellm/proxy/spend_tracking/auto_router_sessions.py index 682ad178bf1..3b65fa98728 100644 --- a/litellm/proxy/spend_tracking/auto_router_sessions.py +++ b/litellm/proxy/spend_tracking/auto_router_sessions.py @@ -28,7 +28,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger -from litellm.router_utils.auto_router_model_naming import StrategyRouterKind, classify_strategy_router_model +from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model if TYPE_CHECKING: from litellm.router import Router @@ -289,35 +289,28 @@ def _unclassified(turn: TurnFacts, baseline_spend: float, state: SessionState) - ) -def auto_router_kind(router: "Router", model_group: str) -> StrategyRouterKind | None: - """Which kind of auto-router serves ``model_group``, or None when none does. +def serves_an_auto_router(router: "Router", model_group: str) -> bool: + """Whether any pre-routing strategy is registered under ``model_group``. - Answered from the registries the router already keys by public alias, so the - lookup costs the same on a proxy with a thousand models as on one with ten. - It runs on every request that carries a session, most of which never touched - an auto-router, so it must decide that before paying for anything else; a - scan of ``model_list`` charged the whole proxy for a feature few requests use. + A gate, not a classifier. It runs on every request that carries a session, + most of which never touched an auto-router, so it answers from the registries + the router already keys by public alias and costs the same on a proxy with a + thousand models as on one with ten. Scanning ``model_list`` instead charged + the whole proxy for a feature few requests use. - The alias is load-bearing rather than incidental: the auto-router's own - classifier sub-calls share the session but carry the judge model's group, so - keying on it folds one turn per routed request and no classifier noise. - - One deployment can own an entry in more than one registry; a complexity - router with ``adaptive`` enabled is in both. The order below is therefore the - order ``classify_strategy_router_model`` resolves its prefixes in, which is - what ``auto_router_group_kinds`` labels the same group with on the read side. - The two have to agree, or the dashboard would label a group one way and the - rows it aggregates another. + Which kind of router ran is deliberately not answered here. One alias can own + entries in several registries, and which of them served a given request + depends on the request's tags; only ``Router._select_pre_routing_strategy`` + resolves that, and it already recorded the answer on the routing decision. A + second derivation here would disagree with it on exactly the tagged + deployments the first one exists to disambiguate. """ - if model_group in router.complexity_routers: - return "complexity" - if model_group in router.adaptive_routers: - return "adaptive" - if model_group in router.quality_routers: - return "quality" - if model_group in router.auto_routers: - return "semantic" - return None + return ( + model_group in router.complexity_routers + or model_group in router.adaptive_routers + or model_group in router.quality_routers + or model_group in router.auto_routers + ) def auto_router_group_kinds(router: "Router") -> Mapping[str, str]: diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index 4ea3389381c..0c280fe36ae 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -125,6 +125,7 @@ class AutoRouter(CustomLogger): LiteLLMRouterEncoder, ) from litellm.types.router import PreRoutingHookResponse + from litellm.types.utils import StandardLoggingRoutingDecision if messages is None: # do nothing, return same inputs @@ -148,12 +149,21 @@ class AutoRouter(CustomLogger): message_content = self._extract_text_from_messages(messages) route_choice: RouteChoice | list[RouteChoice] | None = routelayer(text=message_content) verbose_router_logger.debug("route_choice: %s", route_choice) + matched: str | None = None if isinstance(route_choice, RouteChoice): - model = route_choice.name or self.default_model + matched = route_choice.name + model = matched or self.default_model elif isinstance(route_choice, list): - model = route_choice[0].name or self.default_model + matched = route_choice[0].name + model = matched or self.default_model return PreRoutingHookResponse( model=model, messages=messages, + routing_decision=StandardLoggingRoutingDecision( + router_model_name=self.model_name, + router_type="semantic", + routed_model=model, + cause="semantic_keyword_match" if matched else "default_fallback", + ), ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index b2b29fc34c4..4d78a39a597 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2790,7 +2790,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): """Per-request provenance for a pre-routing strategy (auto-router) decision.""" router_model_name: str - router_type: Literal["complexity", "adaptive", "quality"] + router_type: Literal["complexity", "adaptive", "quality", "semantic"] routed_model: str cause: RoutingDecisionCause tier: str diff --git a/tests/test_litellm/proxy/spend_tracking/test_auto_router_benchmarks.py b/tests/test_litellm/proxy/spend_tracking/test_auto_router_benchmarks.py index d402655bd34..39279c0083e 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_auto_router_benchmarks.py +++ b/tests/test_litellm/proxy/spend_tracking/test_auto_router_benchmarks.py @@ -42,6 +42,7 @@ class _FakePrisma: def group_row(**overrides): row = { "model_group": "claude-auto", + "router_kind": "semantic", "baseline_model": "anthropic/claude-opus-4-8", "sessions": 10, "turns": 100, @@ -204,9 +205,10 @@ class TestReadPathSource: assert len(prisma.db.queries) == 1 assert {g.model_group for g in result.groups} == {"claude-auto", "claude-router-2"} - async def test_each_group_is_labelled_with_its_router_kind(self): - prisma = _FakePrisma([group_row(model_group="claude-router-2")]) - result = await compute_benchmarks(prisma, {"claude-router-2": "complexity"}, START, END) + async def test_a_group_is_labelled_with_the_kind_its_rows_recorded(self): + """The rows know which strategy served the turns; the config only knows what is registered.""" + prisma = _FakePrisma([group_row(model_group="claude-router-2", router_kind="complexity")]) + result = await compute_benchmarks(prisma, {"claude-router-2": "semantic"}, START, END) assert result.groups[0].router_kind == "complexity" diff --git a/tests/test_litellm/proxy/spend_tracking/test_auto_router_sessions.py b/tests/test_litellm/proxy/spend_tracking/test_auto_router_sessions.py index d214ae5a337..c99f3f0e21f 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_auto_router_sessions.py +++ b/tests/test_litellm/proxy/spend_tracking/test_auto_router_sessions.py @@ -843,26 +843,20 @@ class _RegistriesOnly: self.auto_routers = dict.fromkeys(semantic, ()) -class TestResolvingOneGroupsKind: - @pytest.mark.parametrize( - "registry, kind", - [ - ("complexity", "complexity"), - ("quality", "quality"), - ("adaptive", "adaptive"), - ("semantic", "semantic"), - ], - ) - def test_every_kind_of_auto_router_is_resolved_from_its_own_registry(self, registry, kind): - from litellm.proxy.spend_tracking.auto_router_sessions import auto_router_kind +class TestTheGateRecognisesAnAutoRouterGroup: + """The write path gates on membership only. Which strategy ran is the router's + answer, recorded on the routing decision, not something re-derived here.""" - router = _RegistriesOnly(**{registry: ("a-router",)}) - assert auto_router_kind(router, "a-router") == kind + @pytest.mark.parametrize("registry", ["complexity", "quality", "adaptive", "semantic"]) + def test_a_group_in_any_registry_passes_the_gate(self, registry): + from litellm.proxy.spend_tracking.auto_router_sessions import serves_an_auto_router - def test_a_plain_model_group_resolves_to_nothing(self): - from litellm.proxy.spend_tracking.auto_router_sessions import auto_router_kind + assert serves_an_auto_router(_RegistriesOnly(**{registry: ("a-router",)}), "a-router") - assert auto_router_kind(_RegistriesOnly(complexity=("a-router",)), "gpt-4o") is None + def test_a_plain_model_group_does_not(self): + from litellm.proxy.spend_tracking.auto_router_sessions import serves_an_auto_router + + assert not serves_an_auto_router(_RegistriesOnly(complexity=("a-router",)), "gpt-4o") class TestTheWriteAndReadPathsAgreeOnWhatAnAutoRouterIs: @@ -871,14 +865,14 @@ class TestTheWriteAndReadPathsAgreeOnWhatAnAutoRouterIs: dashboard labels another, or filters out entirely, is a benchmark that reads empty for traffic that really happened.""" - def test_every_group_the_dashboard_asks_about_resolves_to_the_kind_it_labels(self): - from litellm.proxy.spend_tracking.auto_router_sessions import auto_router_group_kinds, auto_router_kind + def test_every_group_the_dashboard_asks_about_passes_the_write_paths_gate(self): + from litellm.proxy.spend_tracking.auto_router_sessions import auto_router_group_kinds, serves_an_auto_router router = _router_with_auto_routers() group_kinds = auto_router_group_kinds(router) assert dict(group_kinds) == {"adaptive-complexity-router": "complexity", "quality-router": "quality"} - assert {group: auto_router_kind(router, group) for group in group_kinds} == dict(group_kinds) + assert all(serves_an_auto_router(router, group) for group in group_kinds) def test_a_complexity_router_running_the_bandit_is_still_a_complexity_router(self): """It owns an entry in both registries, so the lookup order decides, and only @@ -889,9 +883,9 @@ class TestTheWriteAndReadPathsAgreeOnWhatAnAutoRouterIs: assert "adaptive-complexity-router" in router.adaptive_routers def test_a_group_no_auto_router_serves_is_left_out_of_both(self): - from litellm.proxy.spend_tracking.auto_router_sessions import auto_router_group_kinds, auto_router_kind + from litellm.proxy.spend_tracking.auto_router_sessions import auto_router_group_kinds, serves_an_auto_router router = _router_with_auto_routers() assert "cheap" not in auto_router_group_kinds(router) - assert auto_router_kind(router, "cheap") is None + assert not serves_an_auto_router(router, "cheap")