diff --git a/litellm/router.py b/litellm/router.py index 7267f6eb3ba..d721260d90a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9279,6 +9279,7 @@ class Router: config=config, model_to_prefs=model_to_prefs, model_to_cost=model_to_cost, + litellm_router_instance=self, ) self._register_pre_routing_strategy( registry=self.adaptive_routers, diff --git a/litellm/router_strategy/adaptive_router/adaptive_router.py b/litellm/router_strategy/adaptive_router/adaptive_router.py index 9aa881fc4c9..dfd9c43e0a1 100644 --- a/litellm/router_strategy/adaptive_router/adaptive_router.py +++ b/litellm/router_strategy/adaptive_router/adaptive_router.py @@ -18,7 +18,7 @@ import time from collections import OrderedDict from collections.abc import Mapping from dataclasses import asdict, dataclass -from typing import Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, cast from litellm._logging import verbose_router_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -51,8 +51,15 @@ from litellm.router_strategy.adaptive_router.signals import ( from litellm.router_strategy.adaptive_router.update_queue import ( AdaptiveRouterUpdateQueue, ) +from litellm.router_strategy.savings_baseline import ( + conversation_is_continuing, + resolve_baseline, +) from litellm.types.utils import StandardLoggingRoutingDecision +if TYPE_CHECKING: + from litellm.router import Router + # Sweep session-state cache when it exceeds this many live entries. Expired # entries are dropped in bulk; amortizes to O(1) per insert. _SESSION_STATE_SWEEP_THRESHOLD: Final[int] = 1024 @@ -92,11 +99,13 @@ class AdaptiveRouter: config: AdaptiveRouterConfig, model_to_prefs: dict[str, AdaptiveRouterPreferences], model_to_cost: dict[str, float], + litellm_router_instance: Router | None = None, ) -> None: self.router_name = router_name self.config = config self.model_to_prefs = model_to_prefs self.model_to_cost = model_to_cost + self.litellm_router_instance = litellm_router_instance self.queue = AdaptiveRouterUpdateQueue() self._cells: dict[tuple[RequestType, str], BanditCell] = {} @@ -205,18 +214,35 @@ class AdaptiveRouter: if isinstance(kwargs_metadata, dict): kwargs_metadata[ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY] = chosen_model + routing_decision: Final = StandardLoggingRoutingDecision( + router_model_name=self.router_name, + router_type="adaptive", + routed_model=chosen_model, + cause="bandit", + request_type=request_type.value, + conversation_continuing=conversation_is_continuing(messages), + ) + routing_decision.update(self._savings_baseline_fields()) return PreRoutingHookResponse( model=chosen_model, messages=messages, - routing_decision=StandardLoggingRoutingDecision( - router_model_name=self.router_name, - router_type="adaptive", - routed_model=chosen_model, - cause="bandit", - request_type=request_type.value, - ), + routing_decision=routing_decision, ) + def _savings_baseline_fields(self) -> StandardLoggingRoutingDecision: + if self.litellm_router_instance is None: + return {} # mutable-ok: immutable empty result for absent router + baseline = resolve_baseline(self.litellm_router_instance, self.config.available_models) + if baseline is None: + return {} # mutable-ok: immutable empty result for unresolved baseline + fields: Final[StandardLoggingRoutingDecision] = { + "savings_baseline_model": baseline.model, + **( + {"savings_baseline_deployment_id": baseline.deployment_id} if baseline.deployment_id is not None else {} + ), + } + return fields + # ---- Pick model ------------------------------------------------------ async def pick_model( diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 1f4285a7960..9982a4daab6 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -2945,6 +2945,7 @@ class ComplexityRouter(CustomLogger): ), model_to_prefs=model_to_prefs, model_to_cost=model_to_cost, + litellm_router_instance=self.litellm_router_instance, ) self._adaptive_chosen_model_key = ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY return self.adaptive_router diff --git a/litellm/router_strategy/quality_router/quality_router.py b/litellm/router_strategy/quality_router/quality_router.py index e99d473d455..02f5b5a87b0 100644 --- a/litellm/router_strategy/quality_router/quality_router.py +++ b/litellm/router_strategy/quality_router/quality_router.py @@ -24,6 +24,10 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.router_strategy.complexity_router.complexity_router import ( ComplexityRouter, ) +from litellm.router_strategy.savings_baseline import ( + conversation_is_continuing, + resolve_baseline, +) from litellm.types.utils import StandardLoggingRoutingDecision from .config import QualityRouterConfig, RoutingPreferences @@ -319,6 +323,18 @@ class QualityRouter(CustomLogger): if isinstance(metadata, dict): metadata["quality_router_decision"] = decision + def _savings_fields(self) -> StandardLoggingRoutingDecision: + baseline = resolve_baseline(self.litellm_router_instance, self.config.available_models) + if baseline is None: + return {} # mutable-ok: immutable empty result for unresolved baseline + fields: Final[StandardLoggingRoutingDecision] = { + "savings_baseline_model": baseline.model, + **( + {"savings_baseline_deployment_id": baseline.deployment_id} if baseline.deployment_id is not None else {} + ), + } + return fields + async def async_pre_routing_hook( self, model: str, @@ -334,6 +350,9 @@ class QualityRouter(CustomLogger): verbose_router_logger.debug("QualityRouter: No messages provided, skipping routing") return None + conversation_continuing: Final = conversation_is_continuing(messages) + savings_fields: Final = self._savings_fields() + # Extract last user message and last system prompt — same rules as # ComplexityRouter.async_pre_routing_hook. user_message: str | None = None @@ -357,15 +376,18 @@ class QualityRouter(CustomLogger): verbose_router_logger.debug("QualityRouter: No user message found, routing to default model") if not self.config.default_model: raise ValueError("QualityRouter: no user message and no default_model configured") + default_routing_decision: Final = StandardLoggingRoutingDecision( + router_model_name=self.model_name, + router_type="quality", + routed_model=self.config.default_model, + cause="default_fallback", + conversation_continuing=conversation_continuing, + ) + default_routing_decision.update(savings_fields) return PreRoutingHookResponse( model=self.config.default_model, messages=messages, - routing_decision=StandardLoggingRoutingDecision( - router_model_name=self.model_name, - router_type="quality", - routed_model=self.config.default_model, - cause="default_fallback", - ), + routing_decision=default_routing_decision, ) # Try keyword override first — it short-circuits complexity classification. @@ -388,22 +410,25 @@ class QualityRouter(CustomLogger): "matched_keyword": matched_keyword, "quality_tier": self._model_quality.get(routed_model), "complexity_tier": None, + "conversation_continuing": conversation_continuing, }, ) - routing_decision: Final = StandardLoggingRoutingDecision( + keyword_routing_decision: Final = StandardLoggingRoutingDecision( router_model_name=self.model_name, router_type="quality", routed_model=routed_model, cause="keyword", matched_keyword=matched_keyword, + conversation_continuing=conversation_continuing, ) + keyword_routing_decision.update(savings_fields) keyword_quality_tier: Final = self._model_quality.get(routed_model) if keyword_quality_tier is not None: - routing_decision["tier"] = str(keyword_quality_tier) + keyword_routing_decision["tier"] = str(keyword_quality_tier) return PreRoutingHookResponse( model=routed_model, messages=messages, - routing_decision=routing_decision, + routing_decision=keyword_routing_decision, ) # No keyword match → complexity classification flow. @@ -434,19 +459,23 @@ class QualityRouter(CustomLogger): "matched_keyword": None, "quality_tier": int(quality_tier), "complexity_tier": complexity_name, + "conversation_continuing": conversation_continuing, }, ) + quality_routing_decision: Final = StandardLoggingRoutingDecision( + router_model_name=self.model_name, + router_type="quality", + routed_model=routed_model, + cause="quality_tier", + tier=str(int(quality_tier)), + score=score, + signals=list(signals), # mutable-ok: routing decision metadata uses a JSON list + conversation_continuing=conversation_continuing, + ) + quality_routing_decision.update(savings_fields) return PreRoutingHookResponse( model=routed_model, messages=messages, - routing_decision=StandardLoggingRoutingDecision( - router_model_name=self.model_name, - router_type="quality", - routed_model=routed_model, - cause="quality_tier", - tier=str(int(quality_tier)), - score=score, - signals=list(signals), - ), + routing_decision=quality_routing_decision, ) diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index a2e983a8369..0e17025679d 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -14,7 +14,7 @@ bare string with no provider beside them; an operator who writes ``deepseek-r1`` Azure would otherwise be priced against whoever else owns that name. """ -from collections.abc import Iterable +from collections.abc import Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Final, NamedTuple from litellm._logging import verbose_router_logger @@ -45,6 +45,11 @@ class Baseline(NamedTuple): deployment_id: str | None = None +def conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None) -> bool: + """Return whether the request contains evidence of an earlier assistant turn.""" + return not messages or any(message.get("role") == "assistant" for message in messages) + + def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | None: """``provider/model``, or ``None`` when the pair names no known provider. diff --git a/tests/test_litellm/router_strategy/adaptive_router/test_async_pre_routing.py b/tests/test_litellm/router_strategy/adaptive_router/test_async_pre_routing.py index 0a8c230e773..9e0efc05247 100644 --- a/tests/test_litellm/router_strategy/adaptive_router/test_async_pre_routing.py +++ b/tests/test_litellm/router_strategy/adaptive_router/test_async_pre_routing.py @@ -7,7 +7,7 @@ model on metadata, and return a PreRoutingHookResponse. Routing is stateless per-turn — `pick_model` does not take a session id. """ -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import pytest @@ -43,6 +43,51 @@ async def test_returns_pre_routing_hook_response_with_chosen_model(): assert response.model == "smart" +@pytest.mark.asyncio +async def test_records_savings_baseline_and_conversation_shape(): + router = _make_router() + router_instance = MagicMock() + router_instance.model_name_to_deployment_indices = {"fast": [0], "smart": [1]} + router_instance.model_list = [ + {"litellm_params": {"model": "openai/fast"}, "model_info": {"id": "fast-id"}}, + {"litellm_params": {"model": "openai/smart"}, "model_info": {"id": "smart-id"}}, + ] + router_instance.get_deployment_model_info.side_effect = lambda deployment_id, model: { + "input_cost_per_token": 0.00000015 if deployment_id == "fast-id" else 0.000005, + "output_cost_per_token": 0.00000015 if deployment_id == "fast-id" else 0.000005, + } + router.litellm_router_instance = router_instance + router.pick_model = AsyncMock(return_value="smart") # type: ignore[method-assign] + + response = await router.async_pre_routing_hook( + model="smart-cheap-router", + request_kwargs={}, + messages=[ + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "answer"}, + {"role": "user", "content": "next"}, + ], + ) + + assert response is not None + assert response.routing_decision["conversation_continuing"] is True + assert response.routing_decision["savings_baseline_model"] == "openai/smart" + assert response.routing_decision["savings_baseline_deployment_id"] == "smart-id" + + +@pytest.mark.asyncio +async def test_empty_messages_use_conservative_continuing_shape(): + router = _make_router() + router.pick_model = AsyncMock(return_value="smart") # type: ignore[method-assign] + + response = await router.async_pre_routing_hook( + model="smart-cheap-router", request_kwargs={}, messages=None + ) + + assert response is not None + assert response.routing_decision["conversation_continuing"] is True + + @pytest.mark.asyncio async def test_classifies_last_user_message_for_request_type(): r = _make_router() diff --git a/tests/test_litellm/router_strategy/test_quality_router.py b/tests/test_litellm/router_strategy/test_quality_router.py index 4e87652f8b3..be984bcb5a1 100644 --- a/tests/test_litellm/router_strategy/test_quality_router.py +++ b/tests/test_litellm/router_strategy/test_quality_router.py @@ -822,6 +822,70 @@ class TestKeywordOverride: class TestDecisionMetadata: + @pytest.mark.asyncio + @pytest.mark.parametrize("content, cause", [("hi", "quality_tier"), ("write python", "keyword")]) + async def test_conversation_shape_is_read_once_for_metadata_and_savings(self, keyword_router, content, cause): + class CountedMessage(dict): + role_reads = 0 + + def get(self, key, default=None): + if key == "role": + self.role_reads += 1 + return super().get(key, default) + + message = CountedMessage(role="user", content=content) + kwargs: Dict[str, Any] = {} + response = await keyword_router.async_pre_routing_hook("qr", kwargs, [message]) + + assert response is not None + assert response.routing_decision["cause"] == cause + assert response.routing_decision["conversation_continuing"] is False + assert kwargs["metadata"]["quality_router_decision"]["conversation_continuing"] is False + assert message.role_reads == 2 + + @pytest.mark.asyncio + async def test_decision_includes_savings_baseline_and_conversation_shape(self, quality_router): + quality_router.litellm_router_instance.model_name_to_deployment_indices = { + "haiku": [0], + "sonnet": [1], + "opus": [2], + "opus-next": [3], + } + quality_router.litellm_router_instance.get_deployment_model_info.side_effect = ( + lambda deployment_id, model: { + "input_cost_per_token": { + "id-haiku": 0.000001, + "id-sonnet": 0.000002, + "id-opus": 0.000003, + "id-opus-next": 0.000004, + }[deployment_id], + "output_cost_per_token": 0.000001, + } + ) + request_kwargs: Dict[str, Any] = {} + response = await quality_router.async_pre_routing_hook( + model="quality-router-test", + request_kwargs=request_kwargs, + messages=[ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "continue"}, + ], + ) + + assert response is not None + assert response.routing_decision["conversation_continuing"] is True + assert response.routing_decision["savings_baseline_model"] == "openai/sonnet" + assert response.routing_decision["savings_baseline_deployment_id"] == "id-sonnet" + + first_turn = await quality_router.async_pre_routing_hook( + model="quality-router-test", + request_kwargs={}, + messages=[{"role": "user", "content": "first request"}], + ) + assert first_turn is not None + assert first_turn.routing_decision["conversation_continuing"] is False + @pytest.mark.asyncio async def test_hook_stashes_decision_in_request_kwargs_metadata( self, quality_router diff --git a/tests/test_litellm/router_strategy/test_savings_baseline.py b/tests/test_litellm/router_strategy/test_savings_baseline.py index 5efc73d2dcb..859482edc08 100644 --- a/tests/test_litellm/router_strategy/test_savings_baseline.py +++ b/tests/test_litellm/router_strategy/test_savings_baseline.py @@ -4,6 +4,7 @@ from litellm.router import Router from litellm.router_strategy.savings_baseline import ( Baseline, canonical_model, + conversation_is_continuing, _models_in, _most_expensive, resolve_baseline, @@ -22,6 +23,19 @@ def parent() -> Router: ) +@pytest.mark.parametrize( + "messages, expected", + [ + (None, True), + ([], True), + ([{"role": "user", "content": "hello"}], False), + ([{"role": "user"}, {"role": "assistant"}, {"role": "user"}], True), + ], +) +def test_conversation_shape_for_savings(messages, expected): + assert conversation_is_continuing(messages) is expected + + class TestCanonicalModel: def test_qualifies_a_bare_name_with_the_provider_that_owns_it(self): assert canonical_model("claude-opus-5") == "anthropic/claude-opus-5"