From aa2426365126048beb1ba75104fdddecd67001b3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:09:05 -0700 Subject: [PATCH 1/5] fix(router): let untagged requests bypass a tagged pre-routing strategy on shared model names --- litellm/router.py | 22 +++++- .../router_strategy/test_complexity_router.py | 35 +++++++++ tests/test_litellm/test_router.py | 76 +++++++++++++++++++ 3 files changed, 131 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 8917ef60e36..ac1a3101f01 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -11339,11 +11339,25 @@ class Router: return filtered + def _model_name_has_plain_deployments(self, model: str) -> bool: + """True when `model` also names regular (non strategy-router) deployments in the model_list.""" + indices: Final = self.model_name_to_deployment_indices.get(model) or () + return any( + classify_strategy_router_model(lp.get("model") or "") is None + for idx in indices + if (lp := self.model_list[idx].get("litellm_params")) + ) + def _select_pre_routing_strategy(self, model: str, request_kwargs: dict) -> "PreRoutingStrategy | None": """ Resolve the pre-routing strategy for `model`, disambiguating deployments that share a `model_name` by matching the request's tags against each registered strategy's tags before falling back to the first registered. + + With tag filtering enabled, strategies that all carry real tags matching + none of the request's do not capture it when the name also has plain + deployments: returning None hands the request to ordinary tag-aware + deployment selection. """ candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [ *self.auto_routers.get(model, []), @@ -11353,8 +11367,6 @@ class Router: ] if not candidates: return None - if len(candidates) == 1: - return candidates[0].strategy request_tags: Final = _get_tags_from_request_kwargs(request_kwargs) if request_tags: @@ -11366,6 +11378,12 @@ class Router: for tagged in candidates: if "default" in tagged.tags: return tagged.strategy + if ( + self.enable_tag_filtering + and all(tagged.tags for tagged in candidates) + and self._model_name_has_plain_deployments(model) + ): + return None return candidates[0].strategy async def async_pre_routing_hook( diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 94b6b68855b..8abe80ca0d7 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1176,6 +1176,41 @@ class TestPreRoutingStrategyRegistry: } assert router._select_pre_routing_strategy("smart", {}) is cn + @staticmethod + def _router_with_plain_smart_deployment(enable_tag_filtering: bool) -> Router: + return Router( + model_list=[{"model_name": "smart", "litellm_params": {"model": "openai/gpt-4o-mini"}}], + enable_tag_filtering=enable_tag_filtering, + ) + + def test_select_falls_through_to_plain_deployments_when_no_tag_matches_under_tag_filtering(self): + router = self._router_with_plain_smart_deployment(enable_tag_filtering=True) + cn, us = object(), object() + + router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]} + assert router._select_pre_routing_strategy("smart", {}) is None + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}) is cn + + router.complexity_routers = { + "smart": [ + TaggedPreRoutingStrategy(tags=("cn",), strategy=cn), + TaggedPreRoutingStrategy(tags=("us",), strategy=us), + ] + } + assert router._select_pre_routing_strategy("smart", {}) is None + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["row"]}}) is None + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}) is us + + router.complexity_routers["router-only"] = [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)] + assert router._select_pre_routing_strategy("router-only", {}) is cn + + def test_select_keeps_capturing_when_tag_filtering_is_disabled(self): + router = self._router_with_plain_smart_deployment(enable_tag_filtering=False) + cn = object() + + router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]} + assert router._select_pre_routing_strategy("smart", {}) is cn + class TestAsyncPreRoutingHookMultiFormat: """Test async_pre_routing_hook with multiple input formats.""" diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 0a16b998f82..48931bd2fe1 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7521,6 +7521,82 @@ class TestAutoRouterMaxInputCharsWiring: assert self._registered_auto_router(router).max_input_chars == DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS +class TestTaggedAutoRouterOnSharedModelName: + """A tagged auto-router marker sharing its model_name with a plain deployment must not + capture requests whose tags don't match it when tag filtering is enabled (#36620).""" + + class _FixedRouteLayer: + def __call__(self, text: str): + from semantic_router.schema import RouteChoice + + return RouteChoice(name="gemini-flash") + + @classmethod + def _router(cls, marker_tags, include_plain_sibling: bool, enable_tag_filtering: bool) -> "litellm.Router": + pytest.importorskip("semantic_router", reason="auto-router needs the semantic-router extra") + marker = { + "model_name": "gpt4o", + "litellm_params": { + "model": "auto_router/gpt4o-router", + "auto_router_config": json.dumps( + {"routes": [{"name": "gemini-flash", "utterances": ["capital city questions"]}]} + ), + "auto_router_default_model": "gemini-flash", + "auto_router_embedding_model": "text-embedding-3-small", + **({"tags": marker_tags} if marker_tags else {}), + }, + } + plain = {"model_name": "gpt4o", "litellm_params": {"model": "openai/gpt-4o"}} + tier = {"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}} + router = litellm.Router( + model_list=[plain, marker, tier] if include_plain_sibling else [marker, tier], + enable_tag_filtering=enable_tag_filtering, + ) + router.auto_routers["gpt4o"][0].strategy.routelayer = cls._FixedRouteLayer() + return router + + @staticmethod + async def _hook_response(router: "litellm.Router", request_kwargs: dict): + return await router.async_pre_routing_hook( + model="gpt4o", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + + @pytest.mark.asyncio + async def test_untagged_request_bypasses_the_tagged_marker_when_a_plain_deployment_shares_the_name(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + + assert await self._hook_response(router, {}) is None + + @pytest.mark.asyncio + async def test_request_tagged_for_the_marker_is_still_semantically_routed(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + + response = await self._hook_response(router, {"metadata": {"tags": ["route"]}}) + + assert response is not None + assert response.model == "gemini-flash" + + @pytest.mark.asyncio + async def test_marker_only_alias_still_captures_untagged_requests(self): + router = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True) + + response = await self._hook_response(router, {}) + + assert response is not None + assert response.model == "gemini-flash" + + @pytest.mark.asyncio + async def test_untagged_marker_sharing_the_name_still_captures_untagged_requests(self): + router = self._router(marker_tags=None, include_plain_sibling=True, enable_tag_filtering=True) + + response = await self._hook_response(router, {}) + + assert response is not None + assert response.model == "gemini-flash" + + class TestGetAllowedFailsFromPolicy: def _make_router(self, **policy_kwargs) -> litellm.Router: from litellm.types.router import AllowedFailsPolicy From bcba392b214977b5e7cf416b46edcf20f9637722 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:44:13 -0700 Subject: [PATCH 2/5] fix(router): exclude strategy marker deployments from selection when plain siblings exist --- litellm/router.py | 22 ++++++++++++++++------ tests/test_litellm/test_router.py | 12 ++++++++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index ac1a3101f01..60e93a0526c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10699,6 +10699,15 @@ class Router: return None + @staticmethod + def _is_strategy_marker_deployment(deployment: Mapping[str, object]) -> bool: + """True when the deployment is a strategy-router pseudo-model (`auto_router/` prefixed).""" + litellm_params: Final = deployment.get("litellm_params") + if not isinstance(litellm_params, Mapping): + return False + deployment_model: Final = litellm_params.get("model") + return isinstance(deployment_model, str) and classify_strategy_router_model(deployment_model) is not None + def _common_checks_available_deployment( self, model: str, @@ -10826,7 +10835,12 @@ class Router: model ] # update the model to the actual value if an alias has been passed in - return model, healthy_deployments + marker_flags: Final = tuple(self._is_strategy_marker_deployment(d) for d in healthy_deployments) + if all(marker_flags) or not any(marker_flags): + return model, healthy_deployments + return model, [ # mutable-ok: matches this function's list contract expected by downstream filters + d for d, is_marker in zip(healthy_deployments, marker_flags, strict=True) if not is_marker + ] def _filter_deployments_by_model_access_groups( self, @@ -11342,11 +11356,7 @@ class Router: def _model_name_has_plain_deployments(self, model: str) -> bool: """True when `model` also names regular (non strategy-router) deployments in the model_list.""" indices: Final = self.model_name_to_deployment_indices.get(model) or () - return any( - classify_strategy_router_model(lp.get("model") or "") is None - for idx in indices - if (lp := self.model_list[idx].get("litellm_params")) - ) + return any(not self._is_strategy_marker_deployment(self.model_list[idx]) for idx in indices) def _select_pre_routing_strategy(self, model: str, request_kwargs: dict) -> "PreRoutingStrategy | None": """ diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 48931bd2fe1..640bd229b49 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7596,6 +7596,18 @@ class TestTaggedAutoRouterOnSharedModelName: assert response is not None assert response.model == "gemini-flash" + @pytest.mark.asyncio + async def test_untagged_selection_never_lands_on_the_marker_deployment(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + + for _ in range(20): + deployment = await router.async_get_available_deployment( + model="gpt4o", + request_kwargs={}, + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + assert deployment["litellm_params"]["model"] == "openai/gpt-4o" + class TestGetAllowedFailsFromPolicy: def _make_router(self, **policy_kwargs) -> litellm.Router: From b7136243c7e53eed208f6b455ceb2211b2e32b80 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:17:52 -0700 Subject: [PATCH 3/5] test(router): cover the non-mapping litellm_params marker guard and drop redundant docstrings --- litellm/router.py | 2 -- tests/test_litellm/test_router.py | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 60e93a0526c..1eb2995f558 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10701,7 +10701,6 @@ class Router: @staticmethod def _is_strategy_marker_deployment(deployment: Mapping[str, object]) -> bool: - """True when the deployment is a strategy-router pseudo-model (`auto_router/` prefixed).""" litellm_params: Final = deployment.get("litellm_params") if not isinstance(litellm_params, Mapping): return False @@ -11354,7 +11353,6 @@ class Router: return filtered def _model_name_has_plain_deployments(self, model: str) -> bool: - """True when `model` also names regular (non strategy-router) deployments in the model_list.""" indices: Final = self.model_name_to_deployment_indices.get(model) or () return any(not self._is_strategy_marker_deployment(self.model_list[idx]) for idx in indices) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 640bd229b49..70c13600014 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7608,6 +7608,9 @@ class TestTaggedAutoRouterOnSharedModelName: ) assert deployment["litellm_params"]["model"] == "openai/gpt-4o" + def test_deployment_without_litellm_params_mapping_is_not_a_marker(self): + assert litellm.Router._is_strategy_marker_deployment({"model_name": "gpt4o"}) is False + class TestGetAllowedFailsFromPolicy: def _make_router(self, **policy_kwargs) -> litellm.Router: From 0f6e5abd491d391a336b8252db0c4da58c97a862 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:28:13 -0700 Subject: [PATCH 4/5] test(router): reference _model_name_has_plain_deployments directly for the router coverage gate --- tests/test_litellm/test_router.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 70c13600014..fe0d97b08b0 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7611,6 +7611,13 @@ class TestTaggedAutoRouterOnSharedModelName: def test_deployment_without_litellm_params_mapping_is_not_a_marker(self): assert litellm.Router._is_strategy_marker_deployment({"model_name": "gpt4o"}) is False + def test_model_name_has_plain_deployments_reflects_the_pool(self): + mixed = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + marker_only = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True) + + assert mixed._model_name_has_plain_deployments("gpt4o") is True + assert marker_only._model_name_has_plain_deployments("gpt4o") is False + class TestGetAllowedFailsFromPolicy: def _make_router(self, **policy_kwargs) -> litellm.Router: From ce3d30a43414f5b719fcbd78d92ba1794e50ecb6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 13 Aug 2026 00:45:49 +0000 Subject: [PATCH 5/5] fix(router): read no tags, instead of raising, from a non-dict metadata bucket Pre-routing now reads the request's tags on every request with a registered strategy, including the single-strategy case that used to short-circuit before looking at tags. Metadata is request-controlled, so a caller that sends `litellm_metadata` (or `tags`) as a string or any other non-dict shape crashed tag lookup with an AttributeError instead of routing untagged. --- litellm/litellm_core_utils/core_helpers.py | 4 +-- litellm/router_strategy/tag_based_routing.py | 35 ++++++++++++++----- .../test_router_tag_routing.py | 27 ++++++++++++++ 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 2462c282041..de1092bc02f 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -1,7 +1,7 @@ # What is this? ## Helper utilities import copy -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Final, Literal import httpx @@ -181,7 +181,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad def get_metadata_variable_name_from_kwargs( - kwargs: dict, + kwargs: Mapping[str, object], ) -> Literal["metadata", "litellm_metadata"]: """ Helper to return what the "metadata" field should be called in the request data diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index 40ed89ebfd8..1120323b4f9 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -584,8 +584,26 @@ async def get_deployments_for_tag( return healthy_deployments +def _tags_in_metadata(metadata: object) -> list[str]: + """ + Tags out of a metadata bucket the caller controls the shape of. + + A request can send its metadata (and its ``tags``) as anything the JSON body + allowed, an unparsed string or null included, so any shape that is not a list + of string tags carries no tags rather than raising. + """ + if not isinstance(metadata, Mapping): + return [] + typed_metadata: Final[Mapping[str, object]] = metadata + tags: Final = typed_metadata.get("tags") + if isinstance(tags, str) or not isinstance(tags, Sequence): + return [] + typed_tags: Final[Sequence[object]] = tags + return [tag for tag in typed_tags if isinstance(tag, str)] + + def _get_tags_from_request_kwargs( - request_kwargs: dict[Any, Any] | None = None, + request_kwargs: Mapping[Any, Any] | None = None, metadata_variable_name: Literal["metadata", "litellm_metadata"] | None = None, ) -> list[str]: """ @@ -604,12 +622,11 @@ def _get_tags_from_request_kwargs( return [] resolved_variable_name: Final = metadata_variable_name or get_metadata_variable_name_from_kwargs(request_kwargs) if resolved_variable_name in request_kwargs: - metadata: Final = request_kwargs[resolved_variable_name] or {} - tags = metadata.get("tags", []) - return tags if tags is not None else [] - elif "litellm_params" in request_kwargs: - litellm_params: Final = request_kwargs["litellm_params"] or {} - _metadata: Final = litellm_params.get(resolved_variable_name, {}) or {} - tags = _metadata.get("tags", []) - return tags if tags is not None else [] + return _tags_in_metadata(request_kwargs[resolved_variable_name]) + if "litellm_params" in request_kwargs: + litellm_params: Final = request_kwargs["litellm_params"] + if not isinstance(litellm_params, Mapping): + return [] + typed_litellm_params: Final[Mapping[str, object]] = litellm_params + return _tags_in_metadata(typed_litellm_params.get(resolved_variable_name)) return [] diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index 93011bb29cc..73491490b14 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -423,6 +423,33 @@ def test_get_tags_from_request_kwargs_various_inputs(): assert _get_tags_from_request_kwargs({"foo": "bar"}) == [] +@pytest.mark.parametrize( + "request_kwargs", + [ + {"metadata": "not-a-dict"}, + {"litellm_metadata": "not-a-dict"}, + {"litellm_metadata": ["not", "a", "dict"]}, + {"litellm_params": "not-a-dict"}, + {"litellm_params": {"metadata": "not-a-dict"}}, + {"metadata": {"tags": "free"}}, + {"metadata": {"tags": {"free": "paid"}}}, + ], +) +def test_get_tags_from_request_kwargs_reads_no_tags_from_a_non_dict_shape(request_kwargs): + """Metadata and `tags` are request-controlled, so a client can send either as a + string, a list or null. Every shape that cannot hold string tags reads as untagged + instead of raising, because callers run on the hot request path.""" + from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs + + assert _get_tags_from_request_kwargs(request_kwargs) == [] + + +def test_get_tags_from_request_kwargs_keeps_only_string_tags(): + from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs + + assert _get_tags_from_request_kwargs({"metadata": {"tags": ["free", 7, None, "paid"]}}) == ["free", "paid"] + + # --- _split_tags unit tests ---