mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(router): let untagged requests bypass a tagged pre-routing strategy on shared model names
This commit is contained in:
parent
7a55ca811b
commit
aa24263651
3 changed files with 131 additions and 2 deletions
|
|
@ -11339,11 +11339,25 @@ class Router:
|
|||
|
||||
return filtered
|
||||
|
||||
def _model_name_has_plain_deployments(self, model: str) -> bool:
|
||||
"""True when `model` also names regular (non strategy-router) deployments in the model_list."""
|
||||
indices: Final = self.model_name_to_deployment_indices.get(model) or ()
|
||||
return any(
|
||||
classify_strategy_router_model(lp.get("model") or "") is None
|
||||
for idx in indices
|
||||
if (lp := self.model_list[idx].get("litellm_params"))
|
||||
)
|
||||
|
||||
def _select_pre_routing_strategy(self, model: str, request_kwargs: dict) -> "PreRoutingStrategy | None":
|
||||
"""
|
||||
Resolve the pre-routing strategy for `model`, disambiguating deployments
|
||||
that share a `model_name` by matching the request's tags against each
|
||||
registered strategy's tags before falling back to the first registered.
|
||||
|
||||
With tag filtering enabled, strategies that all carry real tags matching
|
||||
none of the request's do not capture it when the name also has plain
|
||||
deployments: returning None hands the request to ordinary tag-aware
|
||||
deployment selection.
|
||||
"""
|
||||
candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [
|
||||
*self.auto_routers.get(model, []),
|
||||
|
|
@ -11353,8 +11367,6 @@ class Router:
|
|||
]
|
||||
if not candidates:
|
||||
return None
|
||||
if len(candidates) == 1:
|
||||
return candidates[0].strategy
|
||||
|
||||
request_tags: Final = _get_tags_from_request_kwargs(request_kwargs)
|
||||
if request_tags:
|
||||
|
|
@ -11366,6 +11378,12 @@ class Router:
|
|||
for tagged in candidates:
|
||||
if "default" in tagged.tags:
|
||||
return tagged.strategy
|
||||
if (
|
||||
self.enable_tag_filtering
|
||||
and all(tagged.tags for tagged in candidates)
|
||||
and self._model_name_has_plain_deployments(model)
|
||||
):
|
||||
return None
|
||||
return candidates[0].strategy
|
||||
|
||||
async def async_pre_routing_hook(
|
||||
|
|
|
|||
|
|
@ -1176,6 +1176,41 @@ class TestPreRoutingStrategyRegistry:
|
|||
}
|
||||
assert router._select_pre_routing_strategy("smart", {}) is cn
|
||||
|
||||
@staticmethod
|
||||
def _router_with_plain_smart_deployment(enable_tag_filtering: bool) -> Router:
|
||||
return Router(
|
||||
model_list=[{"model_name": "smart", "litellm_params": {"model": "openai/gpt-4o-mini"}}],
|
||||
enable_tag_filtering=enable_tag_filtering,
|
||||
)
|
||||
|
||||
def test_select_falls_through_to_plain_deployments_when_no_tag_matches_under_tag_filtering(self):
|
||||
router = self._router_with_plain_smart_deployment(enable_tag_filtering=True)
|
||||
cn, us = object(), object()
|
||||
|
||||
router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]}
|
||||
assert router._select_pre_routing_strategy("smart", {}) is None
|
||||
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}) is cn
|
||||
|
||||
router.complexity_routers = {
|
||||
"smart": [
|
||||
TaggedPreRoutingStrategy(tags=("cn",), strategy=cn),
|
||||
TaggedPreRoutingStrategy(tags=("us",), strategy=us),
|
||||
]
|
||||
}
|
||||
assert router._select_pre_routing_strategy("smart", {}) is None
|
||||
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["row"]}}) is None
|
||||
assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}) is us
|
||||
|
||||
router.complexity_routers["router-only"] = [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]
|
||||
assert router._select_pre_routing_strategy("router-only", {}) is cn
|
||||
|
||||
def test_select_keeps_capturing_when_tag_filtering_is_disabled(self):
|
||||
router = self._router_with_plain_smart_deployment(enable_tag_filtering=False)
|
||||
cn = object()
|
||||
|
||||
router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]}
|
||||
assert router._select_pre_routing_strategy("smart", {}) is cn
|
||||
|
||||
|
||||
class TestAsyncPreRoutingHookMultiFormat:
|
||||
"""Test async_pre_routing_hook with multiple input formats."""
|
||||
|
|
|
|||
|
|
@ -7521,6 +7521,82 @@ class TestAutoRouterMaxInputCharsWiring:
|
|||
assert self._registered_auto_router(router).max_input_chars == DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS
|
||||
|
||||
|
||||
class TestTaggedAutoRouterOnSharedModelName:
|
||||
"""A tagged auto-router marker sharing its model_name with a plain deployment must not
|
||||
capture requests whose tags don't match it when tag filtering is enabled (#36620)."""
|
||||
|
||||
class _FixedRouteLayer:
|
||||
def __call__(self, text: str):
|
||||
from semantic_router.schema import RouteChoice
|
||||
|
||||
return RouteChoice(name="gemini-flash")
|
||||
|
||||
@classmethod
|
||||
def _router(cls, marker_tags, include_plain_sibling: bool, enable_tag_filtering: bool) -> "litellm.Router":
|
||||
pytest.importorskip("semantic_router", reason="auto-router needs the semantic-router extra")
|
||||
marker = {
|
||||
"model_name": "gpt4o",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/gpt4o-router",
|
||||
"auto_router_config": json.dumps(
|
||||
{"routes": [{"name": "gemini-flash", "utterances": ["capital city questions"]}]}
|
||||
),
|
||||
"auto_router_default_model": "gemini-flash",
|
||||
"auto_router_embedding_model": "text-embedding-3-small",
|
||||
**({"tags": marker_tags} if marker_tags else {}),
|
||||
},
|
||||
}
|
||||
plain = {"model_name": "gpt4o", "litellm_params": {"model": "openai/gpt-4o"}}
|
||||
tier = {"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}}
|
||||
router = litellm.Router(
|
||||
model_list=[plain, marker, tier] if include_plain_sibling else [marker, tier],
|
||||
enable_tag_filtering=enable_tag_filtering,
|
||||
)
|
||||
router.auto_routers["gpt4o"][0].strategy.routelayer = cls._FixedRouteLayer()
|
||||
return router
|
||||
|
||||
@staticmethod
|
||||
async def _hook_response(router: "litellm.Router", request_kwargs: dict):
|
||||
return await router.async_pre_routing_hook(
|
||||
model="gpt4o",
|
||||
request_kwargs=request_kwargs,
|
||||
messages=[{"role": "user", "content": "What is the capital of France?"}],
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_untagged_request_bypasses_the_tagged_marker_when_a_plain_deployment_shares_the_name(self):
|
||||
router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True)
|
||||
|
||||
assert await self._hook_response(router, {}) is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_tagged_for_the_marker_is_still_semantically_routed(self):
|
||||
router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True)
|
||||
|
||||
response = await self._hook_response(router, {"metadata": {"tags": ["route"]}})
|
||||
|
||||
assert response is not None
|
||||
assert response.model == "gemini-flash"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_marker_only_alias_still_captures_untagged_requests(self):
|
||||
router = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True)
|
||||
|
||||
response = await self._hook_response(router, {})
|
||||
|
||||
assert response is not None
|
||||
assert response.model == "gemini-flash"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_untagged_marker_sharing_the_name_still_captures_untagged_requests(self):
|
||||
router = self._router(marker_tags=None, include_plain_sibling=True, enable_tag_filtering=True)
|
||||
|
||||
response = await self._hook_response(router, {})
|
||||
|
||||
assert response is not None
|
||||
assert response.model == "gemini-flash"
|
||||
|
||||
|
||||
class TestGetAllowedFailsFromPolicy:
|
||||
def _make_router(self, **policy_kwargs) -> litellm.Router:
|
||||
from litellm.types.router import AllowedFailsPolicy
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue