fix(router): tier-pinned reasoning_effort supersedes client effort carriers (#38698)

An auto-router tier pin injects reasoning_effort into request_kwargs, but
provider translations give a caller-supplied thinking, output_config.effort,
or reasoning carrier precedence over the reasoning_effort alias, so the pin
never reached the wire whenever the client expressed effort natively. Drop
the client's other encodings of the setting at the tier-param merge; a
client output_config keeps its non-effort fields
This commit is contained in:
tin-berri 2026-08-28 14:53:33 -07:00 committed by GitHub
parent fb80ba7c98
commit 42d278cfad
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 200 additions and 8 deletions

View file

@ -11952,6 +11952,33 @@ class Router:
return healthy_deployments
@staticmethod
def _pop_effort_from_nested_carrier(request_kwargs: dict[str, object], carrier: str) -> None:
nested: Final = request_kwargs.get(carrier)
if not isinstance(nested, dict):
return
nested.pop("effort", None)
if not nested:
request_kwargs.pop(carrier, None)
@staticmethod
def _drop_client_effort_carriers_a_tier_pin_supersedes(
request_kwargs: dict[str, object],
tier_litellm_params: Mapping[str, object],
) -> None:
"""Tier litellm_params are deliberate operator overrides, but provider
translations let a caller-supplied carrier of the same setting
(``thinking``, ``output_config.effort``, ``reasoning.effort``) outrank
the ``reasoning_effort`` alias, so a pinned effort only reaches the wire
if the client's other encodings are removed before the merge. Non-effort
fields a carrier also holds (``output_config.format``,
``reasoning.summary``) are kept."""
if "reasoning_effort" not in tier_litellm_params:
return
request_kwargs.pop("thinking", None)
Router._pop_effort_from_nested_carrier(request_kwargs, "output_config")
Router._pop_effort_from_nested_carrier(request_kwargs, "reasoning")
async def async_get_available_deployment(
self,
model: str,
@ -11997,11 +12024,11 @@ class Router:
model = pre_routing_hook_response.model
messages = pre_routing_hook_response.messages
if pre_routing_hook_response.litellm_params:
request_kwargs.update(
self._tier_params_the_target_accepts(
model, pre_routing_hook_response.litellm_params, request_kwargs
)
accepted_tier_params: Final = self._tier_params_the_target_accepts(
model, pre_routing_hook_response.litellm_params, request_kwargs
)
self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params)
request_kwargs.update(accepted_tier_params)
#########################################################
# Resolve the strategy and logger AFTER the pre-routing hook, since
@ -12112,11 +12139,11 @@ class Router:
model = pre_routing_hook_response.model
messages = pre_routing_hook_response.messages
if pre_routing_hook_response.litellm_params:
request_kwargs.update(
self._tier_params_the_target_accepts(
model, pre_routing_hook_response.litellm_params, request_kwargs
)
accepted_tier_params: Final = self._tier_params_the_target_accepts(
model, pre_routing_hook_response.litellm_params, request_kwargs
)
self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params)
request_kwargs.update(accepted_tier_params)
# 2. Get healthy deployments
healthy_deployments: Final = await self.async_get_healthy_deployments(

View file

@ -2234,6 +2234,171 @@ class TestRouterPreRoutingAliasOverrides:
assert deployment["model_name"] == "gpt-5-mini"
assert request_kwargs["reasoning_effort"] == "xhigh"
def _make_effort_pinned_router(self, tier_litellm_params: Dict) -> Router:
return Router(
model_list=[
{
"model_name": "smart-router",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"tiers": {
"SIMPLE": {
"model_name": "gpt-5-mini",
"litellm_params": tier_litellm_params,
}
}
},
},
},
{"model_name": "gpt-5-mini", "litellm_params": {"model": "openai/gpt-5-mini"}},
]
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"client_carriers, expected_absent, expected_present",
[
(
{"thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}},
("thinking", "output_config"),
{},
),
({"reasoning": {"effort": "high"}}, ("reasoning",), {}),
(
{"reasoning": {"effort": "high", "summary": "concise"}},
(),
{"reasoning": {"summary": "concise"}},
),
(
{"output_config": {"effort": "max", "format": {"type": "json_schema"}}},
(),
{"output_config": {"format": {"type": "json_schema"}}},
),
],
)
async def test_tier_pinned_effort_supersedes_client_effort_carriers(
self, client_carriers, expected_absent, expected_present
):
"""A tier-pinned reasoning_effort is an operator override, but provider
translations give a caller-supplied thinking/output_config/reasoning
carrier precedence over the reasoning_effort alias, so the pin only
reaches the wire if those carriers are dropped at the merge."""
router = self._make_effort_pinned_router({"reasoning_effort": "xhigh"})
request_kwargs: Dict = dict(client_carriers)
await router.async_get_available_deployment(
model="smart-router",
request_kwargs=request_kwargs,
messages=[{"role": "user", "content": "hi"}],
)
assert request_kwargs["reasoning_effort"] == "xhigh"
for key in expected_absent:
assert key not in request_kwargs
for key, value in expected_present.items():
assert request_kwargs[key] == value
@pytest.mark.asyncio
async def test_tier_pinned_effort_supersedes_client_carriers_on_pass_through_path(self):
router = Router(
model_list=[
{
"model_name": "smart-router",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"tiers": {
"SIMPLE": {
"model_name": "gpt-5-mini",
"litellm_params": {"reasoning_effort": "xhigh"},
}
}
},
},
},
{
"model_name": "gpt-5-mini",
"litellm_params": {"model": "openai/gpt-5-mini", "use_in_pass_through": True},
},
]
)
request_kwargs: Dict = {"thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}}
await router.async_get_available_deployment_for_pass_through(
model="smart-router",
request_kwargs=request_kwargs,
messages=[{"role": "user", "content": "hi"}],
)
assert request_kwargs["reasoning_effort"] == "xhigh"
assert "thinking" not in request_kwargs
assert "output_config" not in request_kwargs
def test_drop_client_effort_carriers_helper_edge_shapes(self):
no_pin: Dict = {"thinking": {"type": "adaptive"}}
Router._drop_client_effort_carriers_a_tier_pin_supersedes(no_pin, {"temperature": 0.1})
assert no_pin == {"thinking": {"type": "adaptive"}}
non_dict_carriers: Dict = {"output_config": "max", "reasoning": 3}
Router._drop_client_effort_carriers_a_tier_pin_supersedes(non_dict_carriers, {"reasoning_effort": "low"})
assert non_dict_carriers == {"output_config": "max", "reasoning": 3}
effort_only: Dict = {"output_config": {"effort": "max"}, "reasoning": {"effort": "high"}}
Router._pop_effort_from_nested_carrier(effort_only, "output_config")
Router._pop_effort_from_nested_carrier(effort_only, "reasoning")
assert effort_only == {}
@pytest.mark.asyncio
async def test_client_effort_carriers_survive_when_gate_drops_the_tier_pin(self):
"""The tier-param gate removes a pin the routed target cannot take, and a
pin that never applies must not strip the client's own effort carriers."""
router = Router(
model_list=[
{
"model_name": "smart-router",
"litellm_params": {
"model": "auto_router/complexity_router",
"complexity_router_config": {
"tiers": {
"SIMPLE": {
"model_name": "gpt-4o-mini",
"litellm_params": {"reasoning_effort": "xhigh"},
}
}
},
},
},
{"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini"}},
]
)
request_kwargs: Dict = {"thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}}
await router.async_get_available_deployment(
model="smart-router",
request_kwargs=request_kwargs,
messages=[{"role": "user", "content": "hi"}],
)
assert "reasoning_effort" not in request_kwargs
assert request_kwargs["thinking"] == {"type": "adaptive"}
assert request_kwargs["output_config"] == {"effort": "max"}
@pytest.mark.asyncio
async def test_client_effort_carriers_survive_when_tier_pins_no_effort(self):
router = self._make_effort_pinned_router({"temperature": 0.2})
request_kwargs: Dict = {"thinking": {"type": "adaptive"}, "output_config": {"effort": "max"}}
await router.async_get_available_deployment(
model="smart-router",
request_kwargs=request_kwargs,
messages=[{"role": "user", "content": "hi"}],
)
assert request_kwargs["thinking"] == {"type": "adaptive"}
assert request_kwargs["output_config"] == {"effort": "max"}
assert request_kwargs["temperature"] == 0.2
@pytest.mark.asyncio
async def test_routing_never_resolves_an_authenticating_provider(self, monkeypatch, tmp_path):
"""Resolving github_copilot runs its OAuth device flow, so the whole routing path must