fix(router): let untagged requests bypass a tagged pre-routing strategy on shared model names

This commit is contained in:
mateo-berri 2026-08-11 23:09:05 -07:00
parent 7a55ca811b
commit aa24263651
3 changed files with 131 additions and 2 deletions

View file

@ -11339,11 +11339,25 @@ class Router:
return filtered
def _model_name_has_plain_deployments(self, model: str) -> bool:
"""True when `model` also names regular (non strategy-router) deployments in the model_list."""
indices: Final = self.model_name_to_deployment_indices.get(model) or ()
return any(
classify_strategy_router_model(lp.get("model") or "") is None
for idx in indices
if (lp := self.model_list[idx].get("litellm_params"))
)
def _select_pre_routing_strategy(self, model: str, request_kwargs: dict) -> "PreRoutingStrategy | None":
"""
Resolve the pre-routing strategy for `model`, disambiguating deployments
that share a `model_name` by matching the request's tags against each
registered strategy's tags before falling back to the first registered.
With tag filtering enabled, strategies that all carry real tags matching
none of the request's do not capture it when the name also has plain
deployments: returning None hands the request to ordinary tag-aware
deployment selection.
"""
candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [
*self.auto_routers.get(model, []),
@ -11353,8 +11367,6 @@ class Router:
]
if not candidates:
return None
if len(candidates) == 1:
return candidates[0].strategy
request_tags: Final = _get_tags_from_request_kwargs(request_kwargs)
if request_tags:
@ -11366,6 +11378,12 @@ class Router:
for tagged in candidates:
if "default" in tagged.tags:
return tagged.strategy
if (
self.enable_tag_filtering
and all(tagged.tags for tagged in candidates)
and self._model_name_has_plain_deployments(model)
):
return None
return candidates[0].strategy
async def async_pre_routing_hook(

View file

@ -1176,6 +1176,41 @@ class TestPreRoutingStrategyRegistry:
}
assert router._select_pre_routing_strategy("smart", {}) is cn
@staticmethod
def _router_with_plain_smart_deployment(enable_tag_filtering: bool) -> Router:
return Router(
model_list=[{"model_name": "smart", "litellm_params": {"model": "openai/gpt-4o-mini"}}],
enable_tag_filtering=enable_tag_filtering,
)
def test_select_falls_through_to_plain_deployments_when_no_tag_matches_under_tag_filtering(self):
router = self._router_with_plain_smart_deployment(enable_tag_filtering=True)
cn, us = object(), object()
router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]}
assert router._select_pre_routing_strategy("smart", {}) is None
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}) is cn
router.complexity_routers = {
"smart": [
TaggedPreRoutingStrategy(tags=("cn",), strategy=cn),
TaggedPreRoutingStrategy(tags=("us",), strategy=us),
]
}
assert router._select_pre_routing_strategy("smart", {}) is None
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["row"]}}) is None
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}) is us
router.complexity_routers["router-only"] = [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]
assert router._select_pre_routing_strategy("router-only", {}) is cn
def test_select_keeps_capturing_when_tag_filtering_is_disabled(self):
router = self._router_with_plain_smart_deployment(enable_tag_filtering=False)
cn = object()
router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]}
assert router._select_pre_routing_strategy("smart", {}) is cn
class TestAsyncPreRoutingHookMultiFormat:
"""Test async_pre_routing_hook with multiple input formats."""

View file

@ -7521,6 +7521,82 @@ class TestAutoRouterMaxInputCharsWiring:
assert self._registered_auto_router(router).max_input_chars == DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS
class TestTaggedAutoRouterOnSharedModelName:
"""A tagged auto-router marker sharing its model_name with a plain deployment must not
capture requests whose tags don't match it when tag filtering is enabled (#36620)."""
class _FixedRouteLayer:
def __call__(self, text: str):
from semantic_router.schema import RouteChoice
return RouteChoice(name="gemini-flash")
@classmethod
def _router(cls, marker_tags, include_plain_sibling: bool, enable_tag_filtering: bool) -> "litellm.Router":
pytest.importorskip("semantic_router", reason="auto-router needs the semantic-router extra")
marker = {
"model_name": "gpt4o",
"litellm_params": {
"model": "auto_router/gpt4o-router",
"auto_router_config": json.dumps(
{"routes": [{"name": "gemini-flash", "utterances": ["capital city questions"]}]}
),
"auto_router_default_model": "gemini-flash",
"auto_router_embedding_model": "text-embedding-3-small",
**({"tags": marker_tags} if marker_tags else {}),
},
}
plain = {"model_name": "gpt4o", "litellm_params": {"model": "openai/gpt-4o"}}
tier = {"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}}
router = litellm.Router(
model_list=[plain, marker, tier] if include_plain_sibling else [marker, tier],
enable_tag_filtering=enable_tag_filtering,
)
router.auto_routers["gpt4o"][0].strategy.routelayer = cls._FixedRouteLayer()
return router
@staticmethod
async def _hook_response(router: "litellm.Router", request_kwargs: dict):
return await router.async_pre_routing_hook(
model="gpt4o",
request_kwargs=request_kwargs,
messages=[{"role": "user", "content": "What is the capital of France?"}],
)
@pytest.mark.asyncio
async def test_untagged_request_bypasses_the_tagged_marker_when_a_plain_deployment_shares_the_name(self):
router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True)
assert await self._hook_response(router, {}) is None
@pytest.mark.asyncio
async def test_request_tagged_for_the_marker_is_still_semantically_routed(self):
router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True)
response = await self._hook_response(router, {"metadata": {"tags": ["route"]}})
assert response is not None
assert response.model == "gemini-flash"
@pytest.mark.asyncio
async def test_marker_only_alias_still_captures_untagged_requests(self):
router = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True)
response = await self._hook_response(router, {})
assert response is not None
assert response.model == "gemini-flash"
@pytest.mark.asyncio
async def test_untagged_marker_sharing_the_name_still_captures_untagged_requests(self):
router = self._router(marker_tags=None, include_plain_sibling=True, enable_tag_filtering=True)
response = await self._hook_response(router, {})
assert response is not None
assert response.model == "gemini-flash"
class TestGetAllowedFailsFromPolicy:
def _make_router(self, **policy_kwargs) -> litellm.Router:
from litellm.types.router import AllowedFailsPolicy