From 42d278cfad9176a4bdbf780275d70d662914c8d2 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Fri, 28 Aug 2026 14:53:33 -0700 Subject: [PATCH] fix(router): tier-pinned reasoning_effort supersedes client effort carriers (#38698) An auto-router tier pin injects reasoning_effort into request_kwargs, but provider translations give a caller-supplied thinking, output_config.effort, or reasoning carrier precedence over the reasoning_effort alias, so the pin never reached the wire whenever the client expressed effort natively. Drop the client's other encodings of the setting at the tier-param merge; a client output_config keeps its non-effort fields --- litellm/router.py | 43 ++++- .../router_strategy/test_complexity_router.py | 165 ++++++++++++++++++ 2 files changed, 200 insertions(+), 8 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 5ed92d845c8..c93c1753f0e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11952,6 +11952,33 @@ class Router: return healthy_deployments + @staticmethod + def _pop_effort_from_nested_carrier(request_kwargs: dict[str, object], carrier: str) -> None: + nested: Final = request_kwargs.get(carrier) + if not isinstance(nested, dict): + return + nested.pop("effort", None) + if not nested: + request_kwargs.pop(carrier, None) + + @staticmethod + def _drop_client_effort_carriers_a_tier_pin_supersedes( + request_kwargs: dict[str, object], + tier_litellm_params: Mapping[str, object], + ) -> None: + """Tier litellm_params are deliberate operator overrides, but provider + translations let a caller-supplied carrier of the same setting + (``thinking``, ``output_config.effort``, ``reasoning.effort``) outrank + the ``reasoning_effort`` alias, so a pinned effort only reaches the wire + if the client's other encodings are removed before the merge. Non-effort + fields a carrier also holds (``output_config.format``, + ``reasoning.summary``) are kept.""" + if "reasoning_effort" not in tier_litellm_params: + return + request_kwargs.pop("thinking", None) + Router._pop_effort_from_nested_carrier(request_kwargs, "output_config") + Router._pop_effort_from_nested_carrier(request_kwargs, "reasoning") + async def async_get_available_deployment( self, model: str, @@ -11997,11 +12024,11 @@ class Router: model = pre_routing_hook_response.model messages = pre_routing_hook_response.messages if pre_routing_hook_response.litellm_params: - request_kwargs.update( - self._tier_params_the_target_accepts( - model, pre_routing_hook_response.litellm_params, request_kwargs - ) + accepted_tier_params: Final = self._tier_params_the_target_accepts( + model, pre_routing_hook_response.litellm_params, request_kwargs ) + self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params) + request_kwargs.update(accepted_tier_params) ######################################################### # Resolve the strategy and logger AFTER the pre-routing hook, since @@ -12112,11 +12139,11 @@ class Router: model = pre_routing_hook_response.model messages = pre_routing_hook_response.messages if pre_routing_hook_response.litellm_params: - request_kwargs.update( - self._tier_params_the_target_accepts( - model, pre_routing_hook_response.litellm_params, request_kwargs - ) + accepted_tier_params: Final = self._tier_params_the_target_accepts( + model, pre_routing_hook_response.litellm_params, request_kwargs ) + self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params) + request_kwargs.update(accepted_tier_params) # 2. Get healthy deployments healthy_deployments: Final = await self.async_get_healthy_deployments( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 0288d4a6318..eee4e9aa185 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -2234,6 +2234,171 @@ class TestRouterPreRoutingAliasOverrides: assert deployment["model_name"] == "gpt-5-mini" assert request_kwargs["reasoning_effort"] == "xhigh" + def _make_effort_pinned_router(self, tier_litellm_params: Dict) -> Router: + return Router( + model_list=[ + { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": { + "SIMPLE": { + "model_name": "gpt-5-mini", + "litellm_params": tier_litellm_params, + } + } + }, + }, + }, + {"model_name": "gpt-5-mini", "litellm_params": {"model": "openai/gpt-5-mini"}}, + ] + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "client_carriers, expected_absent, expected_present", + [ + ( + {"thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}}, + ("thinking", "output_config"), + {}, + ), + ({"reasoning": {"effort": "high"}}, ("reasoning",), {}), + ( + {"reasoning": {"effort": "high", "summary": "concise"}}, + (), + {"reasoning": {"summary": "concise"}}, + ), + ( + {"output_config": {"effort": "max", "format": {"type": "json_schema"}}}, + (), + {"output_config": {"format": {"type": "json_schema"}}}, + ), + ], + ) + async def test_tier_pinned_effort_supersedes_client_effort_carriers( + self, client_carriers, expected_absent, expected_present + ): + """A tier-pinned reasoning_effort is an operator override, but provider + translations give a caller-supplied thinking/output_config/reasoning + carrier precedence over the reasoning_effort alias, so the pin only + reaches the wire if those carriers are dropped at the merge.""" + router = self._make_effort_pinned_router({"reasoning_effort": "xhigh"}) + request_kwargs: Dict = dict(client_carriers) + + await router.async_get_available_deployment( + model="smart-router", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + + assert request_kwargs["reasoning_effort"] == "xhigh" + for key in expected_absent: + assert key not in request_kwargs + for key, value in expected_present.items(): + assert request_kwargs[key] == value + + @pytest.mark.asyncio + async def test_tier_pinned_effort_supersedes_client_carriers_on_pass_through_path(self): + router = Router( + model_list=[ + { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": { + "SIMPLE": { + "model_name": "gpt-5-mini", + "litellm_params": {"reasoning_effort": "xhigh"}, + } + } + }, + }, + }, + { + "model_name": "gpt-5-mini", + "litellm_params": {"model": "openai/gpt-5-mini", "use_in_pass_through": True}, + }, + ] + ) + request_kwargs: Dict = {"thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}} + + await router.async_get_available_deployment_for_pass_through( + model="smart-router", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + + assert request_kwargs["reasoning_effort"] == "xhigh" + assert "thinking" not in request_kwargs + assert "output_config" not in request_kwargs + + def test_drop_client_effort_carriers_helper_edge_shapes(self): + no_pin: Dict = {"thinking": {"type": "adaptive"}} + Router._drop_client_effort_carriers_a_tier_pin_supersedes(no_pin, {"temperature": 0.1}) + assert no_pin == {"thinking": {"type": "adaptive"}} + + non_dict_carriers: Dict = {"output_config": "max", "reasoning": 3} + Router._drop_client_effort_carriers_a_tier_pin_supersedes(non_dict_carriers, {"reasoning_effort": "low"}) + assert non_dict_carriers == {"output_config": "max", "reasoning": 3} + + effort_only: Dict = {"output_config": {"effort": "max"}, "reasoning": {"effort": "high"}} + Router._pop_effort_from_nested_carrier(effort_only, "output_config") + Router._pop_effort_from_nested_carrier(effort_only, "reasoning") + assert effort_only == {} + + @pytest.mark.asyncio + async def test_client_effort_carriers_survive_when_gate_drops_the_tier_pin(self): + """The tier-param gate removes a pin the routed target cannot take, and a + pin that never applies must not strip the client's own effort carriers.""" + router = Router( + model_list=[ + { + "model_name": "smart-router", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "tiers": { + "SIMPLE": { + "model_name": "gpt-4o-mini", + "litellm_params": {"reasoning_effort": "xhigh"}, + } + } + }, + }, + }, + {"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}}, + ] + ) + request_kwargs: Dict = {"thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}} + + await router.async_get_available_deployment( + model="smart-router", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + + assert "reasoning_effort" not in request_kwargs + assert request_kwargs["thinking"] == {"type": "adaptive"} + assert request_kwargs["output_config"] == {"effort": "max"} + + @pytest.mark.asyncio + async def test_client_effort_carriers_survive_when_tier_pins_no_effort(self): + router = self._make_effort_pinned_router({"temperature": 0.2}) + request_kwargs: Dict = {"thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}} + + await router.async_get_available_deployment( + model="smart-router", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "hi"}], + ) + + assert request_kwargs["thinking"] == {"type": "adaptive"} + assert request_kwargs["output_config"] == {"effort": "max"} + assert request_kwargs["temperature"] == 0.2 + @pytest.mark.asyncio async def test_routing_never_resolves_an_authenticating_provider(self, monkeypatch, tmp_path): """Resolving github_copilot runs its OAuth device flow, so the whole routing path must