diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 2462c282041..de1092bc02f 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -1,7 +1,7 @@ # What is this? ## Helper utilities import copy -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Any, Final, Literal import httpx @@ -181,7 +181,7 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad def get_metadata_variable_name_from_kwargs( - kwargs: dict, + kwargs: Mapping[str, object], ) -> Literal["metadata", "litellm_metadata"]: """ Helper to return what the "metadata" field should be called in the request data diff --git a/litellm/router.py b/litellm/router.py index c8441a6c93b..fb2af41dcf2 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10715,6 +10715,14 @@ class Router: return None + @staticmethod + def _is_strategy_marker_deployment(deployment: Mapping[str, object]) -> bool: + litellm_params: Final = deployment.get("litellm_params") + if not isinstance(litellm_params, Mapping): + return False + deployment_model: Final = litellm_params.get("model") + return isinstance(deployment_model, str) and classify_strategy_router_model(deployment_model) is not None + def _common_checks_available_deployment( self, model: str, @@ -10842,7 +10850,12 @@ class Router: model ] # update the model to the actual value if an alias has been passed in - return model, healthy_deployments + marker_flags: Final = tuple(self._is_strategy_marker_deployment(d) for d in healthy_deployments) + if all(marker_flags) or not any(marker_flags): + return model, healthy_deployments + return model, [ # mutable-ok: matches this function's list contract expected by downstream filters + d for d, is_marker in zip(healthy_deployments, marker_flags, strict=True) if not is_marker + ] def _filter_deployments_by_model_access_groups( self, @@ -11355,6 +11368,10 @@ class Router: return filtered + def _model_name_has_plain_deployments(self, model: str) -> bool: + indices: Final = self.model_name_to_deployment_indices.get(model) or () + return any(not self._is_strategy_marker_deployment(self.model_list[idx]) for idx in indices) + def _select_pre_routing_strategy( self, model: str, request_kwargs: dict ) -> "TaggedPreRoutingStrategy[PreRoutingStrategy] | None": @@ -11366,6 +11383,11 @@ class Router: request's tags were what selected it, and can locate the marker deployment the strategy was registered from via its (model_name, tags) pair. + + With tag filtering enabled, strategies that all carry real tags matching + none of the request's do not capture it when the name also has plain + deployments: returning None hands the request to ordinary tag-aware + deployment selection. """ candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [ *self.auto_routers.get(model, []), @@ -11375,8 +11397,6 @@ class Router: ] if not candidates: return None - if len(candidates) == 1: - return candidates[0] request_tags: Final = _get_tags_from_request_kwargs(request_kwargs) if request_tags: @@ -11388,6 +11408,12 @@ class Router: for tagged in candidates: if "default" in tagged.tags: return tagged + if ( + self.enable_tag_filtering + and all(tagged.tags for tagged in candidates) + and self._model_name_has_plain_deployments(model) + ): + return None return candidates[0] async def async_pre_routing_hook( diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index 40ed89ebfd8..1120323b4f9 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -584,8 +584,26 @@ async def get_deployments_for_tag( return healthy_deployments +def _tags_in_metadata(metadata: object) -> list[str]: + """ + Tags out of a metadata bucket the caller controls the shape of. + + A request can send its metadata (and its ``tags``) as anything the JSON body + allowed, an unparsed string or null included, so any shape that is not a list + of string tags carries no tags rather than raising. + """ + if not isinstance(metadata, Mapping): + return [] + typed_metadata: Final[Mapping[str, object]] = metadata + tags: Final = typed_metadata.get("tags") + if isinstance(tags, str) or not isinstance(tags, Sequence): + return [] + typed_tags: Final[Sequence[object]] = tags + return [tag for tag in typed_tags if isinstance(tag, str)] + + def _get_tags_from_request_kwargs( - request_kwargs: dict[Any, Any] | None = None, + request_kwargs: Mapping[Any, Any] | None = None, metadata_variable_name: Literal["metadata", "litellm_metadata"] | None = None, ) -> list[str]: """ @@ -604,12 +622,11 @@ def _get_tags_from_request_kwargs( return [] resolved_variable_name: Final = metadata_variable_name or get_metadata_variable_name_from_kwargs(request_kwargs) if resolved_variable_name in request_kwargs: - metadata: Final = request_kwargs[resolved_variable_name] or {} - tags = metadata.get("tags", []) - return tags if tags is not None else [] - elif "litellm_params" in request_kwargs: - litellm_params: Final = request_kwargs["litellm_params"] or {} - _metadata: Final = litellm_params.get(resolved_variable_name, {}) or {} - tags = _metadata.get("tags", []) - return tags if tags is not None else [] + return _tags_in_metadata(request_kwargs[resolved_variable_name]) + if "litellm_params" in request_kwargs: + litellm_params: Final = request_kwargs["litellm_params"] + if not isinstance(litellm_params, Mapping): + return [] + typed_litellm_params: Final[Mapping[str, object]] = litellm_params + return _tags_in_metadata(typed_litellm_params.get(resolved_variable_name)) return [] diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index ef0e5ac2229..3a69587fdd4 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1176,6 +1176,41 @@ class TestPreRoutingStrategyRegistry: } assert router._select_pre_routing_strategy("smart", {}).strategy is cn + @staticmethod + def _router_with_plain_smart_deployment(enable_tag_filtering: bool) -> Router: + return Router( + model_list=[{"model_name": "smart", "litellm_params": {"model": "openai/gpt-4o-mini"}}], + enable_tag_filtering=enable_tag_filtering, + ) + + def test_select_falls_through_to_plain_deployments_when_no_tag_matches_under_tag_filtering(self): + router = self._router_with_plain_smart_deployment(enable_tag_filtering=True) + cn, us = object(), object() + + router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]} + assert router._select_pre_routing_strategy("smart", {}) is None + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}).strategy is cn + + router.complexity_routers = { + "smart": [ + TaggedPreRoutingStrategy(tags=("cn",), strategy=cn), + TaggedPreRoutingStrategy(tags=("us",), strategy=us), + ] + } + assert router._select_pre_routing_strategy("smart", {}) is None + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["row"]}}) is None + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}).strategy is us + + router.complexity_routers["router-only"] = [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)] + assert router._select_pre_routing_strategy("router-only", {}).strategy is cn + + def test_select_keeps_capturing_when_tag_filtering_is_disabled(self): + router = self._router_with_plain_smart_deployment(enable_tag_filtering=False) + cn = object() + + router.complexity_routers = {"smart": [TaggedPreRoutingStrategy(tags=("cn",), strategy=cn)]} + assert router._select_pre_routing_strategy("smart", {}).strategy is cn + class TestAsyncPreRoutingHookMultiFormat: """Test async_pre_routing_hook with multiple input formats.""" diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index 93011bb29cc..73491490b14 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -423,6 +423,33 @@ def test_get_tags_from_request_kwargs_various_inputs(): assert _get_tags_from_request_kwargs({"foo": "bar"}) == [] +@pytest.mark.parametrize( + "request_kwargs", + [ + {"metadata": "not-a-dict"}, + {"litellm_metadata": "not-a-dict"}, + {"litellm_metadata": ["not", "a", "dict"]}, + {"litellm_params": "not-a-dict"}, + {"litellm_params": {"metadata": "not-a-dict"}}, + {"metadata": {"tags": "free"}}, + {"metadata": {"tags": {"free": "paid"}}}, + ], +) +def test_get_tags_from_request_kwargs_reads_no_tags_from_a_non_dict_shape(request_kwargs): + """Metadata and `tags` are request-controlled, so a client can send either as a + string, a list or null. Every shape that cannot hold string tags reads as untagged + instead of raising, because callers run on the hot request path.""" + from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs + + assert _get_tags_from_request_kwargs(request_kwargs) == [] + + +def test_get_tags_from_request_kwargs_keeps_only_string_tags(): + from litellm.router_strategy.tag_based_routing import _get_tags_from_request_kwargs + + assert _get_tags_from_request_kwargs({"metadata": {"tags": ["free", 7, None, "paid"]}}) == ["free", "paid"] + + # --- _split_tags unit tests --- diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 0b47409eae1..bdbf33fb0e1 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7617,6 +7617,104 @@ class TestAutoRouterMaxInputCharsWiring: assert self._registered_auto_router(router).max_input_chars == DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS +class TestTaggedAutoRouterOnSharedModelName: + """A tagged auto-router marker sharing its model_name with a plain deployment must not + capture requests whose tags don't match it when tag filtering is enabled (#36620).""" + + class _FixedRouteLayer: + def __call__(self, text: str): + from semantic_router.schema import RouteChoice + + return RouteChoice(name="gemini-flash") + + @classmethod + def _router(cls, marker_tags, include_plain_sibling: bool, enable_tag_filtering: bool) -> "litellm.Router": + pytest.importorskip("semantic_router", reason="auto-router needs the semantic-router extra") + marker = { + "model_name": "gpt4o", + "litellm_params": { + "model": "auto_router/gpt4o-router", + "auto_router_config": json.dumps( + {"routes": [{"name": "gemini-flash", "utterances": ["capital city questions"]}]} + ), + "auto_router_default_model": "gemini-flash", + "auto_router_embedding_model": "text-embedding-3-small", + **({"tags": marker_tags} if marker_tags else {}), + }, + } + plain = {"model_name": "gpt4o", "litellm_params": {"model": "openai/gpt-4o"}} + tier = {"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}} + router = litellm.Router( + model_list=[plain, marker, tier] if include_plain_sibling else [marker, tier], + enable_tag_filtering=enable_tag_filtering, + ) + router.auto_routers["gpt4o"][0].strategy.routelayer = cls._FixedRouteLayer() + return router + + @staticmethod + async def _hook_response(router: "litellm.Router", request_kwargs: dict): + return await router.async_pre_routing_hook( + model="gpt4o", + request_kwargs=request_kwargs, + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + + @pytest.mark.asyncio + async def test_untagged_request_bypasses_the_tagged_marker_when_a_plain_deployment_shares_the_name(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + + assert await self._hook_response(router, {}) is None + + @pytest.mark.asyncio + async def test_request_tagged_for_the_marker_is_still_semantically_routed(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + + response = await self._hook_response(router, {"metadata": {"tags": ["route"]}}) + + assert response is not None + assert response.model == "gemini-flash" + + @pytest.mark.asyncio + async def test_marker_only_alias_still_captures_untagged_requests(self): + router = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True) + + response = await self._hook_response(router, {}) + + assert response is not None + assert response.model == "gemini-flash" + + @pytest.mark.asyncio + async def test_untagged_marker_sharing_the_name_still_captures_untagged_requests(self): + router = self._router(marker_tags=None, include_plain_sibling=True, enable_tag_filtering=True) + + response = await self._hook_response(router, {}) + + assert response is not None + assert response.model == "gemini-flash" + + @pytest.mark.asyncio + async def test_untagged_selection_never_lands_on_the_marker_deployment(self): + router = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + + for _ in range(20): + deployment = await router.async_get_available_deployment( + model="gpt4o", + request_kwargs={}, + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + assert deployment["litellm_params"]["model"] == "openai/gpt-4o" + + def test_deployment_without_litellm_params_mapping_is_not_a_marker(self): + assert litellm.Router._is_strategy_marker_deployment({"model_name": "gpt4o"}) is False + + def test_model_name_has_plain_deployments_reflects_the_pool(self): + mixed = self._router(marker_tags=["route"], include_plain_sibling=True, enable_tag_filtering=True) + marker_only = self._router(marker_tags=["route"], include_plain_sibling=False, enable_tag_filtering=True) + + assert mixed._model_name_has_plain_deployments("gpt4o") is True + assert marker_only._model_name_has_plain_deployments("gpt4o") is False + + class TestGetAllowedFailsFromPolicy: def _make_router(self, **policy_kwargs) -> litellm.Router: from litellm.types.router import AllowedFailsPolicy