diff --git a/litellm/router_strategy/savings_baseline.py b/litellm/router_strategy/savings_baseline.py index f13a2868388..0e17025679d 100644 --- a/litellm/router_strategy/savings_baseline.py +++ b/litellm/router_strategy/savings_baseline.py @@ -47,7 +47,7 @@ class Baseline(NamedTuple): def conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None) -> bool: """Return whether the request contains evidence of an earlier assistant turn.""" - return any(message.get("role") == "assistant" for message in messages or ()) + return not messages or any(message.get("role") == "assistant" for message in messages) def canonical_model(model: str, custom_llm_provider: str | None = None) -> str | None: diff --git a/tests/test_litellm/router_strategy/adaptive_router/test_async_pre_routing.py b/tests/test_litellm/router_strategy/adaptive_router/test_async_pre_routing.py index c2efed2e1bf..9e0efc05247 100644 --- a/tests/test_litellm/router_strategy/adaptive_router/test_async_pre_routing.py +++ b/tests/test_litellm/router_strategy/adaptive_router/test_async_pre_routing.py @@ -7,12 +7,11 @@ model on metadata, and return a PreRoutingHookResponse. Routing is stateless per-turn — `pick_model` does not take a session id. """ -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import pytest from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter -from litellm.router_strategy.savings_baseline import Baseline from litellm.types.router import ( AdaptiveRouterConfig, PreRoutingHookResponse, @@ -45,14 +44,20 @@ async def test_returns_pre_routing_hook_response_with_chosen_model(): @pytest.mark.asyncio -async def test_records_savings_baseline_and_conversation_shape(monkeypatch): +async def test_records_savings_baseline_and_conversation_shape(): router = _make_router() - router.litellm_router_instance = object() + router_instance = MagicMock() + router_instance.model_name_to_deployment_indices = {"fast": [0], "smart": [1]} + router_instance.model_list = [ + {"litellm_params": {"model": "openai/fast"}, "model_info": {"id": "fast-id"}}, + {"litellm_params": {"model": "openai/smart"}, "model_info": {"id": "smart-id"}}, + ] + router_instance.get_deployment_model_info.side_effect = lambda deployment_id, model: { + "input_cost_per_token": 0.00000015 if deployment_id == "fast-id" else 0.000005, + "output_cost_per_token": 0.00000015 if deployment_id == "fast-id" else 0.000005, + } + router.litellm_router_instance = router_instance router.pick_model = AsyncMock(return_value="smart") # type: ignore[method-assign] - monkeypatch.setattr( - "litellm.router_strategy.adaptive_router.adaptive_router.resolve_baseline", - lambda router, models: Baseline("openai/smart", "deployment-id"), - ) response = await router.async_pre_routing_hook( model="smart-cheap-router", @@ -67,7 +72,20 @@ async def test_records_savings_baseline_and_conversation_shape(monkeypatch): assert response is not None assert response.routing_decision["conversation_continuing"] is True assert response.routing_decision["savings_baseline_model"] == "openai/smart" - assert response.routing_decision["savings_baseline_deployment_id"] == "deployment-id" + assert response.routing_decision["savings_baseline_deployment_id"] == "smart-id" + + +@pytest.mark.asyncio +async def test_empty_messages_use_conservative_continuing_shape(): + router = _make_router() + router.pick_model = AsyncMock(return_value="smart") # type: ignore[method-assign] + + response = await router.async_pre_routing_hook( + model="smart-cheap-router", request_kwargs={}, messages=None + ) + + assert response is not None + assert response.routing_decision["conversation_continuing"] is True @pytest.mark.asyncio diff --git a/tests/test_litellm/router_strategy/test_quality_router.py b/tests/test_litellm/router_strategy/test_quality_router.py index d85e137b410..98ba9915342 100644 --- a/tests/test_litellm/router_strategy/test_quality_router.py +++ b/tests/test_litellm/router_strategy/test_quality_router.py @@ -19,7 +19,6 @@ from litellm.router_strategy.quality_router.config import ( DEFAULT_COMPLEXITY_TO_QUALITY, ) from litellm.router_strategy.quality_router.quality_router import QualityRouter -from litellm.router_strategy.savings_baseline import Baseline def _make_model_list(spec: List[Dict[str, Any]]) -> List[Dict[str, Any]]: @@ -824,10 +823,23 @@ class TestKeywordOverride: class TestDecisionMetadata: @pytest.mark.asyncio - async def test_decision_includes_savings_baseline_and_conversation_shape(self, quality_router, monkeypatch): - monkeypatch.setattr( - "litellm.router_strategy.quality_router.quality_router.resolve_baseline", - lambda router, models: Baseline("openai/opus-next", "id-opus-next"), + async def test_decision_includes_savings_baseline_and_conversation_shape(self, quality_router): + quality_router.litellm_router_instance.model_name_to_deployment_indices = { + "haiku": [0], + "sonnet": [1], + "opus": [2], + "opus-next": [3], + } + quality_router.litellm_router_instance.get_deployment_model_info.side_effect = ( + lambda deployment_id, model: { + "input_cost_per_token": { + "id-haiku": 0.000001, + "id-sonnet": 0.000002, + "id-opus": 0.000003, + "id-opus-next": 0.000004, + }[deployment_id], + "output_cost_per_token": 0.000001, + } ) request_kwargs: Dict[str, Any] = {} response = await quality_router.async_pre_routing_hook( @@ -842,8 +854,8 @@ class TestDecisionMetadata: assert response is not None assert response.routing_decision["conversation_continuing"] is True - assert response.routing_decision["savings_baseline_model"] == "openai/opus-next" - assert response.routing_decision["savings_baseline_deployment_id"] == "id-opus-next" + assert response.routing_decision["savings_baseline_model"] == "openai/sonnet" + assert response.routing_decision["savings_baseline_deployment_id"] == "id-sonnet" first_turn = await quality_router.async_pre_routing_hook( model="quality-router-test",