diff --git a/litellm/router.py b/litellm/router.py index 8d95e3e91d4..abcdf9068d9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -13136,6 +13136,7 @@ class Router: healthy_deployments=remaining, request_kwargs=tag_request, metadata_variable_name=metadata_variable_name, + request_deployments=request_deployments, ) except ValueError: # tag routing sends this request to none of the rest break @@ -15100,6 +15101,7 @@ class Router: request_kwargs=request_kwargs, healthy_deployments=healthy_deployments, metadata_variable_name=self._get_metadata_variable_name_from_kwargs(request_kwargs), + request_deployments=request_deployments, ) # narrow to whatever `self.routing_plugins` left in candidate_models diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index f13fd3fc630..d3996f504a5 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -9,10 +9,12 @@ Use this to route requests between Teams """ import re -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import replace from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, overload + +from typing_extensions import ReadOnly from litellm._logging import verbose_logger from litellm.constants import ( @@ -56,6 +58,10 @@ class _DeploymentLike(Protocol): def get(self, key: Literal["model_name"], /) -> object: ... +class _DeploymentWithModelInfo(TypedDict, total=False): + model_info: ReadOnly[Mapping[str, object]] + + class _MetadataLike(Protocol): @overload def get(self, key: Literal["tags"], /) -> Sequence[str] | None: ... @@ -285,13 +291,14 @@ def _unknown_required_tag_hides_an_answer( def _chain_allows_fail_open( healthy_deployments: _DeploymentPool, + request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo], excluded_set: frozenset[str], required_set: frozenset[str], routing_confirmed: frozenset[str], ) -> bool: if _unknown_required_tag_hides_an_answer(healthy_deployments, excluded_set, required_set, routing_confirmed): return False - return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in healthy_deployments) + return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in request_deployments) def _trusted_only_pool( @@ -327,6 +334,7 @@ def _trusted_only_pool( def _resolve_or_fail_open( + request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo], pool: Sequence[_DeploymentLike], healthy_deployments: _DeploymentPool, excluded_set: frozenset[str], @@ -339,7 +347,7 @@ def _resolve_or_fail_open( ) -> tuple[_DeploymentLike, ...]: if pool: return tuple(pool) - if _chain_allows_fail_open(healthy_deployments, excluded_set, required_set, routing_confirmed): + if _chain_allows_fail_open(healthy_deployments, request_deployments, excluded_set, required_set, routing_confirmed): # Fall open only within whatever still satisfies whichever constraints # trace back to key/team policy. A constraint with no inherited backing at # all (or, when inherited_tags is unavailable, any constraint at all) can @@ -356,6 +364,7 @@ def _resolve_or_fail_open( def _resolve_constraint_only_pool( + request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo], healthy_deployments: _DeploymentPool, excluded_set: frozenset[str], required_set: frozenset[str], @@ -371,6 +380,7 @@ def _resolve_constraint_only_pool( else _exclude_deployments(_default_tagged_pool(healthy_deployments), excluded_set) ) return _resolve_or_fail_open( + request_deployments, pool, healthy_deployments, excluded_set, @@ -553,6 +563,8 @@ async def get_deployments_for_tag( healthy_deployments: _DeploymentPool, request_kwargs: _RequestKwargsLike | None = None, metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata", + *, + request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo], ) -> _DeploymentPool: """ Returns a list of deployments that match the requested model and tags in the request. @@ -624,6 +636,7 @@ async def get_deployments_for_tag( if constraint_only: return _resolve_constraint_only_pool( + request_deployments, healthy_deployments, excluded_set, required_set, @@ -675,6 +688,7 @@ async def get_deployments_for_tag( if len(new_healthy_deployments) == 0 and len(default_deployments) == 0: return _resolve_or_fail_open( + request_deployments, (), healthy_deployments, excluded_set, @@ -692,6 +706,7 @@ async def get_deployments_for_tag( and _tag_known_to_group(llm_router_instance, model, positive_tags, routing_confirmed) ): return _resolve_or_fail_open( + request_deployments, (), healthy_deployments, excluded_set, diff --git a/tests/unit/router_strategy/test_router_tag_regex_routing.py b/tests/unit/router_strategy/test_router_tag_regex_routing.py index 212c2627d88..46e79a1b8c8 100644 --- a/tests/unit/router_strategy/test_router_tag_regex_routing.py +++ b/tests/unit/router_strategy/test_router_tag_regex_routing.py @@ -121,6 +121,7 @@ async def test_claude_code_ua_routes_to_cc_deployment(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": {"user_agent": "claude-code/1.2.3"}}, ) assert len(result) == 1 @@ -135,6 +136,7 @@ async def test_regular_ua_routes_to_default_deployment(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": {"user_agent": "Mozilla/5.0 (browser)"}}, ) assert len(result) == 1 @@ -149,6 +151,7 @@ async def test_no_ua_routes_to_default_deployment(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": {}}, ) assert len(result) == 1 @@ -164,6 +167,7 @@ async def test_tag_routing_metadata_written_for_regex_match(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": metadata}, ) assert "tag_routing" in metadata @@ -181,6 +185,7 @@ async def test_tag_filtering_disabled_returns_all_deployments(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=ALL_DEPLOYMENTS, + request_deployments=ALL_DEPLOYMENTS, request_kwargs={"metadata": {"user_agent": "claude-code/1.0"}}, ) assert result == ALL_DEPLOYMENTS @@ -199,6 +204,7 @@ async def test_empty_healthy_deployments_with_request_tags_returns_empty_list(): llm_router_instance=router, model="gpt-5.2", healthy_deployments=[], + request_deployments=[], request_kwargs={ "metadata": {"tags": ["client_id:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]} }, @@ -228,6 +234,7 @@ async def test_explicit_tag_match_takes_precedence_over_regex(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=[deployment_with_both], + request_deployments=[deployment_with_both], request_kwargs={"metadata": metadata}, ) assert len(result) == 1 @@ -269,6 +276,7 @@ async def test_user_agent_present_no_tag_regex_deployments_does_not_raise(): llm_router_instance=router, model="gpt-4", healthy_deployments=plain_tag_only_deployments, + request_deployments=plain_tag_only_deployments, request_kwargs={"metadata": {"user_agent": "Mozilla/5.0 (any-client)"}}, ) # Falls through to "return healthy_deployments" path unchanged @@ -305,6 +313,7 @@ async def test_tag_routing_metadata_not_overwritten_for_multiple_matches(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=[deployment_a, deployment_b], + request_deployments=[deployment_a, deployment_b], request_kwargs={"metadata": metadata}, ) assert len(result) == 2 @@ -357,6 +366,7 @@ async def test_match_any_false_strict_tag_check_blocks_regex_fallback(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=[deployment_strict, default_deployment], + request_deployments=[deployment_strict, default_deployment], request_kwargs={"metadata": metadata}, ) ids = [d["model_info"]["id"] for d in result] @@ -387,6 +397,7 @@ async def test_match_any_false_regex_only_deployment_still_matches(): llm_router_instance=router, model="claude-sonnet", healthy_deployments=[regex_only_deployment], + request_deployments=[regex_only_deployment], request_kwargs={"metadata": {"user_agent": "claude-code/1.0"}}, ) assert len(result) == 1 diff --git a/tests/unit/router_strategy/test_router_tag_routing.py b/tests/unit/router_strategy/test_router_tag_routing.py index 097a34cf2e6..26d04be7e25 100644 --- a/tests/unit/router_strategy/test_router_tag_routing.py +++ b/tests/unit/router_strategy/test_router_tag_routing.py @@ -3,12 +3,14 @@ import logging -from typing import Final +from typing import Final, Literal +from unittest.mock import patch import pytest import litellm from litellm._logging import verbose_logger +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_strategy.tag_based_routing import get_deployments_for_tag from litellm.types.router import RouterErrors @@ -1367,14 +1369,17 @@ def test_chain_allows_fail_open_true_when_any_member_sets_flag(): {"model_info": {}, "litellm_params": {"tags": ["provider:anthropic"]}}, {"model_info": {"allow_fail_open": True}, "litellm_params": {"tags": ["provider:openai"]}}, ] - assert _chain_allows_fail_open(deployments, frozenset(), frozenset({"provider:anthropic"}), frozenset()) is True + assert ( + _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset({"provider:anthropic"}), frozenset()) + is True + ) def test_chain_allows_fail_open_false_by_default(): from litellm.router_strategy.tag_based_routing import _chain_allows_fail_open deployments = [{"model_info": {}}, {}] - assert _chain_allows_fail_open(deployments, frozenset(), frozenset(), frozenset()) is False + assert _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset(), frozenset()) is False def test_chain_allows_fail_open_true_when_no_required_tag_is_known_at_all(): @@ -1386,7 +1391,10 @@ def test_chain_allows_fail_open_true_when_no_required_tag_is_known_at_all(): deployments = [ {"model_info": {"allow_fail_open": True}, "litellm_params": {"tags": ["default", "reasoning_type:low"]}}, ] - assert _chain_allows_fail_open(deployments, frozenset(), frozenset({"reasoning_type:high"}), frozenset()) is True + assert ( + _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset({"reasoning_type:high"}), frozenset()) + is True + ) def test_unknown_required_tag_hides_an_answer_denies_fail_open(): @@ -1407,7 +1415,11 @@ def test_unknown_required_tag_hides_an_answer_denies_fail_open(): # fail-open must be denied even though the flag is set on the group. assert ( _chain_allows_fail_open( - deployments, frozenset(), frozenset({"region:us-east", "totally-invented-tag-nobody-has"}), frozenset() + deployments, + deployments, + frozenset(), + frozenset({"region:us-east", "totally-invented-tag-nobody-has"}), + frozenset(), ) is False ) @@ -1434,7 +1446,9 @@ def test_unknown_required_tag_allows_fail_open_when_no_answer_is_hidden(): # carries both, so this is a genuinely unsatisfiable combination, not an invented # tag masking a narrower answer. Fail-open must proceed normally. assert ( - _chain_allows_fail_open(deployments, frozenset(), frozenset({"region:us-east", "region:eu"}), frozenset()) + _chain_allows_fail_open( + deployments, deployments, frozenset(), frozenset({"region:us-east", "region:eu"}), frozenset() + ) is True ) @@ -1496,8 +1510,8 @@ def test_chain_allows_fail_open_true_when_prefixed_unknown_required_tag_is_confi }, ] required_set = frozenset({"provider:anthropic", "typo-tag"}) - assert _chain_allows_fail_open(deployments, frozenset(), required_set, frozenset()) is False - assert _chain_allows_fail_open(deployments, frozenset(), required_set, required_set) is True + assert _chain_allows_fail_open(deployments, deployments, frozenset(), required_set, frozenset()) is False + assert _chain_allows_fail_open(deployments, deployments, frozenset(), required_set, required_set) is True # --- get_deployments_for_tag required-AND ("&") integration tests --- @@ -1765,6 +1779,141 @@ async def test_required_and_combined_with_positive_unmatched_raises_by_default() # --- get_deployments_for_tag allow_fail_open integration tests --- +def _fail_open_router(opted_in_group: str | None = "chat", route: str = "model") -> litellm.Router: + model_name: Final = "chat-*" if route == "wildcard" else "chat" + team_info: Final = {"team_id": "team-a", "team_public_model_name": "public-chat"} if route == "team" else {} + return litellm.Router( + model_list=[ + { + "model_name": model_name if opted_in_group in (None, "chat") else opted_in_group, + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused", "tags": ["vision"]}, + "model_info": {"id": "opted-in", "allow_fail_open": opted_in_group is not None, **team_info}, + }, + { + "model_name": model_name, + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused", "tags": ["region:eu"]}, + "model_info": {"id": "remaining", **team_info}, + }, + ], + routing_groups=( + [{"group_name": "public-chat", "models": ["chat"], "routing_strategy": "simple-shuffle"}] + if route == "routing_group" + else None + ), + model_group_alias={"public-chat": "chat"} if route == "alias" else {}, + enable_tag_filtering=True, + optional_pre_call_checks=["responses_api_deployment_check"], + ) + + +def _response_id_for_remaining_deployment() -> str: + return ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id="remaining", response_id="resp_previous" + ) + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("narrowing", ("affinity", "cooldown")) +@pytest.mark.parametrize("tags", (("component:chat-search",), ("&vision",), ("!region:eu",))) +@pytest.mark.parametrize("route", ("model", "routing_group", "wildcard", "team", "alias")) +async def test_tag_fail_open_survives_deployment_narrowing( + narrowing: Literal["affinity", "cooldown"], tags: tuple[str, ...], route: str +) -> None: + router: Final = _fail_open_router(route=route) + requested_model: Final = "chat" if route == "model" else "chat-test" if route == "wildcard" else "public-chat" + with patch("time.time", return_value=1_700_000_000.0): + if narrowing == "cooldown": + router.cooldown_cache.add_deployment_to_cooldown( + model_id="opted-in", + original_exception=RuntimeError("unavailable"), + exception_status=500, + cooldown_time=60, + ) + candidates: Final = await router.async_get_healthy_deployments( + model=requested_model, + request_kwargs={ + "litellm_metadata": {"tags": list(tags), "inherited_tags": [], "user_api_key_team_id": "team-a"}, + **( + {"previous_response_id": _response_id_for_remaining_deployment()} if narrowing == "affinity" else {} + ), + }, + ) + assert isinstance(candidates, (list, tuple)) + assert tuple(deployment["model_info"]["id"] for deployment in candidates) == ("remaining",) + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("route", ("routing_group", "wildcard", "team", "alias")) +async def test_tag_fail_open_resolves_public_model_names(route: str) -> None: + router: Final = _fail_open_router(route=route) + candidates: Final = await router.async_get_healthy_deployments( + model="chat-test" if route == "wildcard" else "public-chat", + request_kwargs={ + "litellm_metadata": {"tags": ["component:chat-search"], "user_api_key_team_id": "team-a"}, + }, + ) + assert isinstance(candidates, (list, tuple)) + assert {deployment["model_info"]["id"] for deployment in candidates} == {"opted-in", "remaining"} + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("opted_in_group", (None, "other-group")) +async def test_tag_fail_open_requires_opt_in_in_the_requested_group(opted_in_group: str | None) -> None: + router: Final = _fail_open_router(opted_in_group) + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value): + await router.async_get_healthy_deployments( + model="chat", + request_kwargs={ + "previous_response_id": _response_id_for_remaining_deployment(), + "litellm_metadata": {"tags": ["component:chat-search"], "inherited_tags": []}, + }, + ) + + +@pytest.mark.asyncio() +async def test_tag_fail_open_does_not_use_another_teams_opt_in() -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "chat", + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused"}, + "model_info": { + "id": team_id, + "team_id": team_id, + "team_public_model_name": "public-chat", + "allow_fail_open": team_id == "team-a", + }, + } + for team_id in ("team-a", "team-b") + ], + enable_tag_filtering=True, + ) + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value): + await router.async_get_healthy_deployments( + model="public-chat", + request_kwargs={ + "litellm_metadata": {"tags": ["component:chat-search"], "user_api_key_team_id": "team-b"}, + }, + ) + + +@pytest.mark.asyncio() +@pytest.mark.parametrize("inherited_tag", ("&vision", "!region:eu")) +async def test_tag_fail_open_preserves_inherited_policy_after_affinity(inherited_tag: str) -> None: + router: Final = _fail_open_router() + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value): + await router.async_get_healthy_deployments( + model="chat", + request_kwargs={ + "previous_response_id": _response_id_for_remaining_deployment(), + "litellm_metadata": { + "tags": [inherited_tag, "component:chat-search"], + "inherited_tags": [inherited_tag], + }, + }, + ) + + @pytest.mark.asyncio() async def test_allow_fail_open_required_and_unmatched_falls_back_to_default_pool(): router = litellm.Router( @@ -3305,10 +3454,12 @@ async def _routed_to(router: litellm.Router, metadata: dict[str, object]) -> str async def _pool(router: litellm.Router, metadata: dict[str, object]) -> frozenset[str]: + request_deployments: Final = router.get_model_list(model_name="chat") deployments: Final = await get_deployments_for_tag( llm_router_instance=router, model="chat", - healthy_deployments=router.get_model_list(model_name="chat"), + healthy_deployments=request_deployments, + request_deployments=request_deployments, request_kwargs={"metadata": metadata}, ) return frozenset(deployment["model_info"]["id"] for deployment in deployments)