diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 6e3cbdff9d0..7e4cc2d6100 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 23919 + "limit": 23914 }, "reportArgumentType": { "limit": 2580 diff --git a/litellm/constants.py b/litellm/constants.py index 8ab660b0852..554165f5d39 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1325,6 +1325,7 @@ LITELLM_METADATA_FIELD: Final = "litellm_metadata" OLD_LITELLM_METADATA_FIELD: Final = "metadata" RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name" SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl" +CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated" LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = ( diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py index fbf28e223c1..1869328c039 100644 --- a/litellm/proxy/common_utils/callback_utils.py +++ b/litellm/proxy/common_utils/callback_utils.py @@ -6,7 +6,11 @@ from typing import TYPE_CHECKING, Any, Final, Optional import litellm from litellm import get_secret from litellm._logging import verbose_proxy_logger -from litellm.constants import PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY +from litellm.constants import ( + CONSUMED_REQUEST_TAGS_METADATA_KEY, + PRE_CALL_EXECUTED_GUARDRAILS_KEY, + SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, +) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, @@ -426,6 +430,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset( "_pipeline_managed_guardrails", PRE_CALL_EXECUTED_GUARDRAILS_KEY, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + CONSUMED_REQUEST_TAGS_METADATA_KEY, "disable_global_guardrails", "disable_global_guardrail", "opted_out_global_guardrails", diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index f83061a15ce..251ed1feb10 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -16,6 +16,7 @@ import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.constants import ( + CONSUMED_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, LITELLM_PROXY_MASTER_KEY_ALIAS, PRE_CALL_EXECUTED_GUARDRAILS_KEY, @@ -261,6 +262,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( "policy_sources", "routing_decision", SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, + CONSUMED_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, "standard_logging_object", "proxy_server_request", diff --git a/litellm/router.py b/litellm/router.py index 8917ef60e36..ed3c1ae676e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -43,6 +43,7 @@ from litellm.caching.caching import ( RedisClusterCache, ) from litellm.constants import ( + CONSUMED_REQUEST_TAGS_METADATA_KEY, DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, DEFAULT_HEALTH_CHECK_INTERVAL, DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER, @@ -171,6 +172,7 @@ from litellm.types.router import ( AlertingConfig, AllowedFailsPolicy, AssistantsTypedDict, + ConsumedRequestTagsStamp, CredentialLiteLLMParams, CustomRoutingStrategyBase, Deployment, @@ -11339,11 +11341,15 @@ class Router: return filtered - def _select_pre_routing_strategy(self, model: str, request_kwargs: dict) -> "PreRoutingStrategy | None": + def _select_pre_routing_strategy( + self, model: str, request_kwargs: dict + ) -> "TaggedPreRoutingStrategy[PreRoutingStrategy] | None": """ Resolve the pre-routing strategy for `model`, disambiguating deployments that share a `model_name` by matching the request's tags against each registered strategy's tags before falling back to the first registered. + Returns the tagged registry entry so the caller can tell whether the + request's tags were what selected it. """ candidates: Final[list[TaggedPreRoutingStrategy[PreRoutingStrategy]]] = [ *self.auto_routers.get(model, []), @@ -11354,7 +11360,7 @@ class Router: if not candidates: return None if len(candidates) == 1: - return candidates[0].strategy + return candidates[0] request_tags: Final = _get_tags_from_request_kwargs(request_kwargs) if request_tags: @@ -11362,11 +11368,11 @@ class Router: if tagged.tags and is_valid_deployment_tag( list(tagged.tags), request_tags, self.tag_filtering_match_any ): - return tagged.strategy + return tagged for tagged in candidates: if "default" in tagged.tags: - return tagged.strategy - return candidates[0].strategy + return tagged + return candidates[0] async def async_pre_routing_hook( self, @@ -11390,15 +11396,18 @@ class Router: if self.routing_plugins: await self._run_routing_plugins(model=model, request_kwargs=request_kwargs, messages=messages) - router_strategy: Final = self._select_pre_routing_strategy(model=model, request_kwargs=request_kwargs) - if router_strategy is None: + selected_strategy: Final = self._select_pre_routing_strategy(model=model, request_kwargs=request_kwargs) + if selected_strategy is None: self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None) self._stamp_or_clear_metadata_key( request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None ) + self._stamp_or_clear_metadata_key( + request_kwargs=request_kwargs, key=CONSUMED_REQUEST_TAGS_METADATA_KEY, value=None + ) return None - pre_routing_hook_response: Final = await router_strategy.async_pre_routing_hook( + pre_routing_hook_response: Final = await selected_strategy.strategy.async_pre_routing_hook( model=model, request_kwargs=request_kwargs, messages=messages, @@ -11414,6 +11423,15 @@ class Router: key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=(pre_routing_hook_response.session_affinity_ttl_seconds if pre_routing_hook_response else None), ) + self._stamp_or_clear_metadata_key( + request_kwargs=request_kwargs, + key=CONSUMED_REQUEST_TAGS_METADATA_KEY, + value=self._consumed_request_tags_stamp( + selected_strategy=selected_strategy, + pre_routing_hook_response=pre_routing_hook_response, + request_tags=_get_tags_from_request_kwargs(request_kwargs), + ), + ) # `model` (the alias, e.g. "smart-router") is never the deployment actually # called - apply the alias's own litellm_params (besides `model` itself, @@ -11432,6 +11450,28 @@ class Router: return pre_routing_hook_response + def _consumed_request_tags_stamp( + self, + selected_strategy: "TaggedPreRoutingStrategy[PreRoutingStrategy]", + pre_routing_hook_response: PreRoutingHookResponse | None, + request_tags: Sequence[str], + ) -> ConsumedRequestTagsStamp | None: + """Record which tags picked the router and which model group it rewrote to, or None. + + A request whose tags matched the selected strategy's tags has already spent those + tags on picking the router; re-applying them to the routed tier's model group would + empty the pool unless every tier deployment repeats the marker's tag. Only the + strategy's own tags are spent: the request's other tags keep constraining + deployment selection inside the routed group, and key/team policy tags are + untouched because tag filtering separately re-applies whatever + `metadata.inherited_tags` carries for the stamped group. + """ + if pre_routing_hook_response is None or not selected_strategy.tags or not request_tags: + return None + if not is_valid_deployment_tag(selected_strategy.tags, request_tags, self.tag_filtering_match_any): + return None + return ConsumedRequestTagsStamp(model_group=pre_routing_hook_response.model, tags=selected_strategy.tags) + @staticmethod def _record_routing_decision( request_kwargs: dict, diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index bbe97613c57..40ed89ebfd8 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -13,7 +13,9 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal from litellm._logging import verbose_logger -from litellm.types.router import RouterErrors +from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY +from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs +from litellm.types.router import ConsumedRequestTagsStamp, RouterErrors if TYPE_CHECKING: from litellm.router import Router as _Router @@ -46,7 +48,9 @@ def _is_valid_deployment_tag_regex( return None -def is_valid_deployment_tag(deployment_tags: list[str], request_tags: list[str], match_any: bool = True) -> bool: +def is_valid_deployment_tag( + deployment_tags: Sequence[str], request_tags: Sequence[str], match_any: bool = True +) -> bool: """ Check if a tag is valid, the matching can be either any or all based on `match_any` flag """ @@ -389,6 +393,25 @@ def _tag_known_to_group( ) +def _request_tags_after_router_consumption(metadata: Mapping[Any, Any], model: str) -> Sequence[str] | None: + # The pre-routing hook stamps which tags selected the router it rewrote the request + # to: those tags already did their job and must not also constrain deployment choice + # inside the routed group. The request's other tags still apply there, on top of the + # inherited_tags snapshot that keeps key/team policy applying. Every other model + # group keeps the full list. + stamp: Final = metadata.get(CONSUMED_REQUEST_TAGS_METADATA_KEY) + if not isinstance(stamp, ConsumedRequestTagsStamp) or stamp.model_group != model: + return metadata.get("tags") + request_tags: Final = metadata.get("tags") + leftover: Final = tuple( + tag for tag in (request_tags if isinstance(request_tags, (list, tuple)) else ()) if tag not in stamp.tags + ) + inherited_tags: Final = metadata.get("inherited_tags") + if not isinstance(inherited_tags, (list, tuple)): + return leftover or None + return tuple(dict.fromkeys((*leftover, *inherited_tags))) + + async def get_deployments_for_tag( llm_router_instance: LitellmRouter, model: str, # used to raise the correct error @@ -429,7 +452,7 @@ async def get_deployments_for_tag( verbose_logger.debug("request metadata: %s", request_kwargs.get(metadata_variable_name)) if metadata_variable_name in request_kwargs: metadata: Final = request_kwargs[metadata_variable_name] - request_tags: Final = metadata.get("tags") + request_tags: Final = _request_tags_after_router_consumption(metadata, model) match_any: Final = llm_router_instance.tag_filtering_match_any routing_prefix: Final = llm_router_instance.tag_routing_prefix or "" @@ -563,26 +586,30 @@ async def get_deployments_for_tag( def _get_tags_from_request_kwargs( request_kwargs: dict[Any, Any] | None = None, - metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata", + metadata_variable_name: Literal["metadata", "litellm_metadata"] | None = None, ) -> list[str]: """ Helper to get tags from request kwargs Args: request_kwargs: The request kwargs to get tags from + metadata_variable_name: Which metadata dict holds proxy metadata; resolved + from the kwargs when not pinned, so /v1/messages-shaped requests + (``litellm_metadata``) read the same bucket the proxy wrote tags to Returns: List[str]: The tags from the request kwargs """ if request_kwargs is None: return [] - if metadata_variable_name in request_kwargs: - metadata: Final = request_kwargs[metadata_variable_name] or {} + resolved_variable_name: Final = metadata_variable_name or get_metadata_variable_name_from_kwargs(request_kwargs) + if resolved_variable_name in request_kwargs: + metadata: Final = request_kwargs[resolved_variable_name] or {} tags = metadata.get("tags", []) return tags if tags is not None else [] elif "litellm_params" in request_kwargs: litellm_params: Final = request_kwargs["litellm_params"] or {} - _metadata: Final = litellm_params.get(metadata_variable_name, {}) or {} + _metadata: Final = litellm_params.get(resolved_variable_name, {}) or {} tags = _metadata.get("tags", []) return tags if tags is not None else [] return [] diff --git a/litellm/types/router.py b/litellm/types/router.py index d7ff8d12aa6..217364c48b7 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -902,6 +902,14 @@ class TaggedPreRoutingStrategy(Generic[_PreRoutingStrategyT_co]): strategy: _PreRoutingStrategyT_co +@dataclass(frozen=True, slots=True) +class ConsumedRequestTagsStamp: + """The model group a tagged router rewrote to, plus the request tags spent selecting it.""" + + model_group: str + tags: tuple[str, ...] + + @runtime_checkable class PreRoutingStrategy(Protocol): """Structural interface shared by the auto / complexity / adaptive / quality routers.""" diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 94b6b68855b..1308c640a30 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -1157,8 +1157,8 @@ class TestPreRoutingStrategyRegistry: TaggedPreRoutingStrategy(tags=("us",), strategy=us), ] } - assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}) is us - assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}) is cn + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["us"]}}).strategy is us + assert router._select_pre_routing_strategy("smart", {"metadata": {"tags": ["cn"]}}).strategy is cn assert router._select_pre_routing_strategy("missing", {"metadata": {"tags": ["cn"]}}) is None router.complexity_routers = { @@ -1167,14 +1167,14 @@ class TestPreRoutingStrategyRegistry: TaggedPreRoutingStrategy(tags=("default",), strategy=fallback), ] } - assert router._select_pre_routing_strategy("smart", {}) is fallback + assert router._select_pre_routing_strategy("smart", {}).strategy is fallback router.complexity_routers = { "smart": [ TaggedPreRoutingStrategy(tags=("cn",), strategy=cn), TaggedPreRoutingStrategy(tags=("us",), strategy=us), ] } - assert router._select_pre_routing_strategy("smart", {}) is cn + assert router._select_pre_routing_strategy("smart", {}).strategy is cn class TestAsyncPreRoutingHookMultiFormat: diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index 9e19e981f80..93011bb29cc 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -2810,3 +2810,245 @@ def test_update_router_config_schema_includes_tag_routing_prefix(): config = UpdateRouterConfig(tag_routing_prefix="route:") assert config.model_dump(exclude_none=True)["tag_routing_prefix"] == "route:" + + +# --- issue #36621: the request tags that selected a tagged pre-routing strategy +# (e.g. an auto_router marker) are consumed by that selection and must not +# re-apply to the routed tier's model group; key/team-inherited constraints +# must keep applying there --- + + +class _RewriteToTierStrategy: + def __init__(self, rewrite_to: str): + self.rewrite_to = rewrite_to + + async def async_pre_routing_hook( + self, model, request_kwargs, messages=None, input=None, specific_deployment=False + ): + from litellm.types.router import PreRoutingHookResponse + + return PreRoutingHookResponse(model=self.rewrite_to, messages=messages) + + +def _tagged_marker_router(tier_tags=None): + from litellm.types.router import TaggedPreRoutingStrategy + + tier_params = {"model": "gemini/gemini-3.6-flash"} + if tier_tags is not None: + tier_params["tags"] = tier_tags + router = litellm.Router( + model_list=[ + { + "model_name": "gpt4o", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"id": "plain-gpt4o"}, + }, + { + "model_name": "gemini-flash", + "litellm_params": tier_params, + "model_info": {"id": "tier-gemini-flash"}, + }, + ], + enable_tag_filtering=True, + ) + router.auto_routers = { + "gpt4o": [TaggedPreRoutingStrategy(tags=("route",), strategy=_RewriteToTierStrategy("gemini-flash"))] + } + return router + + +@pytest.mark.asyncio() +async def test_router_selecting_tag_is_not_reapplied_to_the_routed_tier(): + # The exact request the auto-router exists to serve: tags=["route"] selects + # the tagged marker, the strategy rewrites to gemini-flash, and the untagged + # tier deployment must serve it instead of 401ing on the already-spent tag. + router = _tagged_marker_router() + + response = await router.acompletion( + model="gpt4o", + messages=[{"role": "user", "content": "What is the capital of France?"}], + metadata={"tags": ["route"], "inherited_tags": []}, + mock_response="Paris", + ) + + assert response._hidden_params["model_id"] == "tier-gemini-flash" + + +@pytest.mark.asyncio() +async def test_router_selecting_tag_is_consumed_on_litellm_metadata_shaped_requests(): + # /v1/messages (and other litellm_metadata endpoints) store proxy metadata, + # including x-litellm-tags header tags, under "litellm_metadata"; consumption + # must read and stamp that same bucket instead of only "metadata". + router = _tagged_marker_router() + + deployment = await router.async_get_available_deployment( + model="gpt4o", + request_kwargs={"litellm_metadata": {"tags": ["route"], "inherited_tags": []}}, + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + + assert deployment["model_info"]["id"] == "tier-gemini-flash" + + +def test_consumed_request_tags_stamp_names_the_routed_group_and_spent_tags_only_on_a_tag_match(): + from litellm.types.router import ConsumedRequestTagsStamp, PreRoutingHookResponse + + router = _tagged_marker_router() + strategy = router.auto_routers["gpt4o"][0] + rewrite = PreRoutingHookResponse(model="gemini-flash", messages=None) + + consumed = router._consumed_request_tags_stamp( + selected_strategy=strategy, pre_routing_hook_response=rewrite, request_tags=["route"] + ) + unmatched = router._consumed_request_tags_stamp( + selected_strategy=strategy, pre_routing_hook_response=rewrite, request_tags=["other"] + ) + + assert consumed == ConsumedRequestTagsStamp(model_group="gemini-flash", tags=("route",)) + assert unmatched is None + + +@pytest.mark.asyncio() +async def test_tagged_request_direct_to_plain_group_still_rejected(): + # Sent straight to the tier, no router selection consumed the tag, so strict + # tag filtering must reject exactly as before. + router = _tagged_marker_router() + + with pytest.raises(Exception) as exc_info: + await router.acompletion( + model="gemini-flash", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["route"], "inherited_tags": []}, + mock_response="hi", + ) + + from litellm.types.router import RouterErrors + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value) + + +@pytest.mark.asyncio() +async def test_caller_forged_consumption_stamp_is_neutralized_by_the_hook(): + # A caller pre-loading the stamp in metadata must not unlock a plain group: + # the pre-routing hook writes-or-clears the stamp on every attempt, and this + # group has no registered strategy, so the forged value is cleared before + # tag filtering runs. + router = _tagged_marker_router() + + with pytest.raises(Exception) as exc_info: + await router.acompletion( + model="gemini-flash", + messages=[{"role": "user", "content": "hi"}], + metadata={ + "tags": ["route"], + "inherited_tags": [], + "_consumed_request_tags": {"model_group": "gemini-flash", "tags": ["route"]}, + }, + mock_response="hi", + ) + + from litellm.types.router import RouterErrors + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value) + + +@pytest.mark.asyncio() +async def test_inherited_constraint_still_applies_to_the_routed_tier(): + # ®ion:eu comes from key/team policy (present in inherited_tags): + # consuming the router-selecting "route" tag must not also discard the + # inherited requirement, so a tier without the tag still raises... + with pytest.raises(Exception) as exc_info: + await _tagged_marker_router().acompletion( + model="gpt4o", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["route", "®ion:eu"], "inherited_tags": ["®ion:eu"]}, + mock_response="hi", + ) + + from litellm.types.router import RouterErrors + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value) + + # ...and a tier carrying it serves the request even though it lacks "route". + response = await _tagged_marker_router(tier_tags=["region:eu"]).acompletion( + model="gpt4o", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["route", "®ion:eu"], "inherited_tags": ["®ion:eu"]}, + mock_response="hi", + ) + + assert response._hidden_params["model_id"] == "tier-gemini-flash" + + +def test_request_tags_after_router_consumption_scopes_to_the_stamped_group(): + from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY + from litellm.router_strategy.tag_based_routing import _request_tags_after_router_consumption + from litellm.types.router import ConsumedRequestTagsStamp + + metadata = { + "tags": ["route", "®ion:eu"], + "inherited_tags": ["®ion:eu"], + CONSUMED_REQUEST_TAGS_METADATA_KEY: ConsumedRequestTagsStamp(model_group="gemini-flash", tags=("route",)), + } + assert _request_tags_after_router_consumption(metadata, "gemini-flash") == ("®ion:eu",) + assert _request_tags_after_router_consumption(metadata, "other-group") == ["route", "®ion:eu"] + + +def test_request_tags_after_router_consumption_drops_only_the_consumed_tags(): + from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY + from litellm.router_strategy.tag_based_routing import _request_tags_after_router_consumption + from litellm.types.router import ConsumedRequestTagsStamp + + fully_consumed = { + "tags": ["route"], + CONSUMED_REQUEST_TAGS_METADATA_KEY: ConsumedRequestTagsStamp(model_group="gemini-flash", tags=("route",)), + } + assert _request_tags_after_router_consumption(fully_consumed, "gemini-flash") is None + + partially_consumed = { + "tags": ["route", "deploy:us"], + "inherited_tags": [], + CONSUMED_REQUEST_TAGS_METADATA_KEY: ConsumedRequestTagsStamp(model_group="gemini-flash", tags=("route",)), + } + assert _request_tags_after_router_consumption(partially_consumed, "gemini-flash") == ("deploy:us",) + + +@pytest.mark.asyncio() +async def test_non_router_tags_still_pick_the_matching_tier_deployment(): + # tags=["route", "deploy:us"]: "route" picks the router and is spent there, + # but "deploy:us" must keep constraining deployment choice inside the routed + # group instead of being dropped with it. + from litellm.types.router import TaggedPreRoutingStrategy + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt4o", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": {"id": "plain-gpt4o"}, + }, + { + "model_name": "gemini-flash", + "litellm_params": {"model": "gemini/gemini-3.6-flash", "tags": ["deploy:us"]}, + "model_info": {"id": "tier-gemini-flash-us"}, + }, + { + "model_name": "gemini-flash", + "litellm_params": {"model": "gemini/gemini-3.6-flash", "tags": ["deploy:eu"]}, + "model_info": {"id": "tier-gemini-flash-eu"}, + }, + ], + enable_tag_filtering=True, + ) + router.auto_routers = { + "gpt4o": [TaggedPreRoutingStrategy(tags=("route",), strategy=_RewriteToTierStrategy("gemini-flash"))] + } + + response = await router.acompletion( + model="gpt4o", + messages=[{"role": "user", "content": "hi"}], + metadata={"tags": ["route", "deploy:us"], "inherited_tags": []}, + mock_response="hi", + ) + + assert response._hidden_params["model_id"] == "tier-gemini-flash-us" diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 0a16b998f82..0b47409eae1 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -7475,6 +7475,102 @@ def test_pre_call_checks_keeps_deployment_when_provider_is_unresolvable(monkeypa assert len(result) == 1 +class TestConsumedRequestTagsStamp: + """Issue #36621: when a request's tags select a tagged pre-routing strategy, those + tags are consumed by the selection; the hook must stamp the rewritten model group so + tag filtering skips request-body tags there, and must clear the stamp on every + re-entry (fallbacks reuse the same request_kwargs) so it cannot leak elsewhere.""" + + class _RewriteStrategy: + def __init__(self, rewrite_to: str): + self.rewrite_to = rewrite_to + + async def async_pre_routing_hook( + self, model, request_kwargs, messages=None, input=None, specific_deployment=False + ): + from litellm.types.router import PreRoutingHookResponse + + return PreRoutingHookResponse(model=self.rewrite_to, messages=messages) + + @classmethod + def _router(cls, marker_tags=("route",)) -> "litellm.Router": + from litellm.types.router import TaggedPreRoutingStrategy + + router = litellm.Router( + model_list=[ + {"model_name": "gpt4o", "litellm_params": {"model": "openai/gpt-4o"}}, + {"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}}, + ], + enable_tag_filtering=True, + ) + router.auto_routers = { + "gpt4o": [TaggedPreRoutingStrategy(tags=marker_tags, strategy=cls._RewriteStrategy("gemini-flash"))] + } + return router + + @pytest.mark.asyncio + async def test_stamps_the_rewritten_group_when_request_tags_selected_the_router(self): + from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY + from litellm.types.router import ConsumedRequestTagsStamp + + router = self._router() + request_kwargs = {"metadata": {"tags": ["route"]}} + + await router.async_pre_routing_hook(model="gpt4o", request_kwargs=request_kwargs) + + assert request_kwargs["metadata"][CONSUMED_REQUEST_TAGS_METADATA_KEY] == ConsumedRequestTagsStamp( + model_group="gemini-flash", tags=("route",) + ) + + @pytest.mark.asyncio + async def test_stamps_into_litellm_metadata_when_the_request_uses_that_bucket(self): + from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY + from litellm.types.router import ConsumedRequestTagsStamp + + router = self._router() + request_kwargs = {"litellm_metadata": {"tags": ["route"]}} + + await router.async_pre_routing_hook(model="gpt4o", request_kwargs=request_kwargs) + + assert request_kwargs["litellm_metadata"][CONSUMED_REQUEST_TAGS_METADATA_KEY] == ConsumedRequestTagsStamp( + model_group="gemini-flash", tags=("route",) + ) + + @pytest.mark.asyncio + async def test_fallback_reentry_with_a_plain_group_clears_the_stale_stamp(self): + from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY + + router = self._router() + request_kwargs = {"metadata": {"tags": ["route"]}} + + await router.async_pre_routing_hook(model="gpt4o", request_kwargs=request_kwargs) + await router.async_pre_routing_hook(model="gemini-flash", request_kwargs=request_kwargs) + + assert CONSUMED_REQUEST_TAGS_METADATA_KEY not in request_kwargs["metadata"] + + @pytest.mark.asyncio + async def test_no_stamp_when_the_request_is_untagged(self): + from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY + + router = self._router() + request_kwargs = {"metadata": {}} + + await router.async_pre_routing_hook(model="gpt4o", request_kwargs=request_kwargs) + + assert CONSUMED_REQUEST_TAGS_METADATA_KEY not in request_kwargs["metadata"] + + @pytest.mark.asyncio + async def test_no_stamp_when_the_selected_strategy_carries_no_tags(self): + from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY + + router = self._router(marker_tags=()) + request_kwargs = {"metadata": {"tags": ["route"]}} + + await router.async_pre_routing_hook(model="gpt4o", request_kwargs=request_kwargs) + + assert CONSUMED_REQUEST_TAGS_METADATA_KEY not in request_kwargs["metadata"] + + class TestAutoRouterMaxInputCharsWiring: """`auto_router_max_input_chars` on the deployment has to reach the AutoRouter that embeds prompts. diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 0237679f2cd..a7286d9a89a 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,6 +1,6 @@ { "LIT001": { - "limit": 23003 + "limit": 23001 }, "LIT002": { "limit": 27146