fix(router): preserve tag fail-open policy through affinity and cooldown (#48470)

litellm/router.py

@@ -13136,6 +13136,7 @@ class Router:
                     healthy_deployments=remaining,
                     request_kwargs=tag_request,
                     metadata_variable_name=metadata_variable_name,
+                    request_deployments=request_deployments,
                 )
             except ValueError:  # tag routing sends this request to none of the rest
                 break
@@ -15100,6 +15101,7 @@ class Router:
             request_kwargs=request_kwargs,
             healthy_deployments=healthy_deployments,
             metadata_variable_name=self._get_metadata_variable_name_from_kwargs(request_kwargs),
+            request_deployments=request_deployments,
         )
 
         # narrow to whatever `self.routing_plugins` left in candidate_models

litellm/router_strategy/tag_based_routing.py

@@ -9,10 +9,12 @@ Use this to route requests between Teams
 """
 
 import re
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterable, Mapping, Sequence
 from dataclasses import replace
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload
+from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, overload
+
+from typing_extensions import ReadOnly
 
 from litellm._logging import verbose_logger
 from litellm.constants import (
@@ -56,6 +58,10 @@ class _DeploymentLike(Protocol):
     def get(self, key: Literal["model_name"], /) -> object: ...
 
 
+class _DeploymentWithModelInfo(TypedDict, total=False):
+    model_info: ReadOnly[Mapping[str, object]]
+
+
 class _MetadataLike(Protocol):
     @overload
     def get(self, key: Literal["tags"], /) -> Sequence[str] | None: ...
@@ -285,13 +291,14 @@ def _unknown_required_tag_hides_an_answer(
 
 def _chain_allows_fail_open(
     healthy_deployments: _DeploymentPool,
+    request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo],
     excluded_set: frozenset[str],
     required_set: frozenset[str],
     routing_confirmed: frozenset[str],
 ) -> bool:
     if _unknown_required_tag_hides_an_answer(healthy_deployments, excluded_set, required_set, routing_confirmed):
         return False
-    return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in healthy_deployments)
+    return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in request_deployments)
 
 
 def _trusted_only_pool(
@@ -327,6 +334,7 @@ def _trusted_only_pool(
 
 
 def _resolve_or_fail_open(
+    request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo],
     pool: Sequence[_DeploymentLike],
     healthy_deployments: _DeploymentPool,
     excluded_set: frozenset[str],
@@ -339,7 +347,7 @@ def _resolve_or_fail_open(
 ) -> tuple[_DeploymentLike, ...]:
     if pool:
         return tuple(pool)
-    if _chain_allows_fail_open(healthy_deployments, excluded_set, required_set, routing_confirmed):
+    if _chain_allows_fail_open(healthy_deployments, request_deployments, excluded_set, required_set, routing_confirmed):
         # Fall open only within whatever still satisfies whichever constraints
         # trace back to key/team policy. A constraint with no inherited backing at
         # all (or, when inherited_tags is unavailable, any constraint at all) can
@@ -356,6 +364,7 @@ def _resolve_or_fail_open(
 
 
 def _resolve_constraint_only_pool(
+    request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo],
     healthy_deployments: _DeploymentPool,
     excluded_set: frozenset[str],
     required_set: frozenset[str],
@@ -371,6 +380,7 @@ def _resolve_constraint_only_pool(
         else _exclude_deployments(_default_tagged_pool(healthy_deployments), excluded_set)
     )
     return _resolve_or_fail_open(
+        request_deployments,
         pool,
         healthy_deployments,
         excluded_set,
@@ -553,6 +563,8 @@ async def get_deployments_for_tag(
     healthy_deployments: _DeploymentPool,
     request_kwargs: _RequestKwargsLike | None = None,
     metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata",
+    *,
+    request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo],
 ) -> _DeploymentPool:
     """
     Returns a list of deployments that match the requested model and tags in the request.
@@ -624,6 +636,7 @@ async def get_deployments_for_tag(
 
         if constraint_only:
             return _resolve_constraint_only_pool(
+                request_deployments,
                 healthy_deployments,
                 excluded_set,
                 required_set,
@@ -675,6 +688,7 @@ async def get_deployments_for_tag(
 
             if len(new_healthy_deployments) == 0 and len(default_deployments) == 0:
                 return _resolve_or_fail_open(
+                    request_deployments,
                     (),
                     healthy_deployments,
                     excluded_set,
@@ -692,6 +706,7 @@ async def get_deployments_for_tag(
                 and _tag_known_to_group(llm_router_instance, model, positive_tags, routing_confirmed)
             ):
                 return _resolve_or_fail_open(
+                    request_deployments,
                     (),
                     healthy_deployments,
                     excluded_set,

tests/unit/router_strategy/test_router_tag_regex_routing.py

@@ -121,6 +121,7 @@ async def test_claude_code_ua_routes_to_cc_deployment():
         llm_router_instance=router,
         model="claude-sonnet",
         healthy_deployments=ALL_DEPLOYMENTS,
+        request_deployments=ALL_DEPLOYMENTS,
         request_kwargs={"metadata": {"user_agent": "claude-code/1.2.3"}},
     )
     assert len(result) == 1
@@ -135,6 +136,7 @@ async def test_regular_ua_routes_to_default_deployment():
         llm_router_instance=router,
         model="claude-sonnet",
         healthy_deployments=ALL_DEPLOYMENTS,
+        request_deployments=ALL_DEPLOYMENTS,
         request_kwargs={"metadata": {"user_agent": "Mozilla/5.0 (browser)"}},
     )
     assert len(result) == 1
@@ -149,6 +151,7 @@ async def test_no_ua_routes_to_default_deployment():
         llm_router_instance=router,
         model="claude-sonnet",
         healthy_deployments=ALL_DEPLOYMENTS,
+        request_deployments=ALL_DEPLOYMENTS,
         request_kwargs={"metadata": {}},
     )
     assert len(result) == 1
@@ -164,6 +167,7 @@ async def test_tag_routing_metadata_written_for_regex_match():
         llm_router_instance=router,
         model="claude-sonnet",
         healthy_deployments=ALL_DEPLOYMENTS,
+        request_deployments=ALL_DEPLOYMENTS,
         request_kwargs={"metadata": metadata},
     )
     assert "tag_routing" in metadata
@@ -181,6 +185,7 @@ async def test_tag_filtering_disabled_returns_all_deployments():
         llm_router_instance=router,
         model="claude-sonnet",
         healthy_deployments=ALL_DEPLOYMENTS,
+        request_deployments=ALL_DEPLOYMENTS,
         request_kwargs={"metadata": {"user_agent": "claude-code/1.0"}},
     )
     assert result == ALL_DEPLOYMENTS
@@ -199,6 +204,7 @@ async def test_empty_healthy_deployments_with_request_tags_returns_empty_list():
         llm_router_instance=router,
         model="gpt-5.2",
         healthy_deployments=[],
+        request_deployments=[],
         request_kwargs={
             "metadata": {"tags": ["client_id:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]}
         },
@@ -228,6 +234,7 @@ async def test_explicit_tag_match_takes_precedence_over_regex():
         llm_router_instance=router,
         model="claude-sonnet",
         healthy_deployments=[deployment_with_both],
+        request_deployments=[deployment_with_both],
         request_kwargs={"metadata": metadata},
     )
     assert len(result) == 1
@@ -269,6 +276,7 @@ async def test_user_agent_present_no_tag_regex_deployments_does_not_raise():
         llm_router_instance=router,
         model="gpt-4",
         healthy_deployments=plain_tag_only_deployments,
+        request_deployments=plain_tag_only_deployments,
         request_kwargs={"metadata": {"user_agent": "Mozilla/5.0 (any-client)"}},
     )
     # Falls through to "return healthy_deployments" path unchanged
@@ -305,6 +313,7 @@ async def test_tag_routing_metadata_not_overwritten_for_multiple_matches():
         llm_router_instance=router,
         model="claude-sonnet",
         healthy_deployments=[deployment_a, deployment_b],
+        request_deployments=[deployment_a, deployment_b],
         request_kwargs={"metadata": metadata},
     )
     assert len(result) == 2
@@ -357,6 +366,7 @@ async def test_match_any_false_strict_tag_check_blocks_regex_fallback():
         llm_router_instance=router,
         model="claude-sonnet",
         healthy_deployments=[deployment_strict, default_deployment],
+        request_deployments=[deployment_strict, default_deployment],
         request_kwargs={"metadata": metadata},
     )
     ids = [d["model_info"]["id"] for d in result]
@@ -387,6 +397,7 @@ async def test_match_any_false_regex_only_deployment_still_matches():
         llm_router_instance=router,
         model="claude-sonnet",
         healthy_deployments=[regex_only_deployment],
+        request_deployments=[regex_only_deployment],
         request_kwargs={"metadata": {"user_agent": "claude-code/1.0"}},
     )
     assert len(result) == 1

tests/unit/router_strategy/test_router_tag_routing.py

@@ -3,12 +3,14 @@
 
 
 import logging
-from typing import Final
+from typing import Final, Literal
+from unittest.mock import patch
 
 import pytest
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.router_strategy.tag_based_routing import get_deployments_for_tag
 from litellm.types.router import RouterErrors
 
@@ -1367,14 +1369,17 @@ def test_chain_allows_fail_open_true_when_any_member_sets_flag():
         {"model_info": {}, "litellm_params": {"tags": ["provider:anthropic"]}},
         {"model_info": {"allow_fail_open": True}, "litellm_params": {"tags": ["provider:openai"]}},
     ]
-    assert _chain_allows_fail_open(deployments, frozenset(), frozenset({"provider:anthropic"}), frozenset()) is True
+    assert (
+        _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset({"provider:anthropic"}), frozenset())
+        is True
+    )
 
 
 def test_chain_allows_fail_open_false_by_default():
     from litellm.router_strategy.tag_based_routing import _chain_allows_fail_open
 
     deployments = [{"model_info": {}}, {}]
-    assert _chain_allows_fail_open(deployments, frozenset(), frozenset(), frozenset()) is False
+    assert _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset(), frozenset()) is False
 
 
 def test_chain_allows_fail_open_true_when_no_required_tag_is_known_at_all():
@@ -1386,7 +1391,10 @@ def test_chain_allows_fail_open_true_when_no_required_tag_is_known_at_all():
     deployments = [
         {"model_info": {"allow_fail_open": True}, "litellm_params": {"tags": ["default", "reasoning_type:low"]}},
     ]
-    assert _chain_allows_fail_open(deployments, frozenset(), frozenset({"reasoning_type:high"}), frozenset()) is True
+    assert (
+        _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset({"reasoning_type:high"}), frozenset())
+        is True
+    )
 
 
 def test_unknown_required_tag_hides_an_answer_denies_fail_open():
@@ -1407,7 +1415,11 @@ def test_unknown_required_tag_hides_an_answer_denies_fail_open():
     # fail-open must be denied even though the flag is set on the group.
     assert (
         _chain_allows_fail_open(
-            deployments, frozenset(), frozenset({"region:us-east", "totally-invented-tag-nobody-has"}), frozenset()
+            deployments,
+            deployments,
+            frozenset(),
+            frozenset({"region:us-east", "totally-invented-tag-nobody-has"}),
+            frozenset(),
         )
         is False
     )
@@ -1434,7 +1446,9 @@ def test_unknown_required_tag_allows_fail_open_when_no_answer_is_hidden():
     # carries both, so this is a genuinely unsatisfiable combination, not an invented
     # tag masking a narrower answer. Fail-open must proceed normally.
     assert (
-        _chain_allows_fail_open(deployments, frozenset(), frozenset({"region:us-east", "region:eu"}), frozenset())
+        _chain_allows_fail_open(
+            deployments, deployments, frozenset(), frozenset({"region:us-east", "region:eu"}), frozenset()
+        )
         is True
     )
 
@@ -1496,8 +1510,8 @@ def test_chain_allows_fail_open_true_when_prefixed_unknown_required_tag_is_confi
         },
     ]
     required_set = frozenset({"provider:anthropic", "typo-tag"})
-    assert _chain_allows_fail_open(deployments, frozenset(), required_set, frozenset()) is False
-    assert _chain_allows_fail_open(deployments, frozenset(), required_set, required_set) is True
+    assert _chain_allows_fail_open(deployments, deployments, frozenset(), required_set, frozenset()) is False
+    assert _chain_allows_fail_open(deployments, deployments, frozenset(), required_set, required_set) is True
 
 
 # --- get_deployments_for_tag required-AND ("&") integration tests ---
@@ -1765,6 +1779,141 @@ async def test_required_and_combined_with_positive_unmatched_raises_by_default()
 # --- get_deployments_for_tag allow_fail_open integration tests ---
 
 
+def _fail_open_router(opted_in_group: str | None = "chat", route: str = "model") -> litellm.Router:
+    model_name: Final = "chat-*" if route == "wildcard" else "chat"
+    team_info: Final = {"team_id": "team-a", "team_public_model_name": "public-chat"} if route == "team" else {}
+    return litellm.Router(
+        model_list=[
+            {
+                "model_name": model_name if opted_in_group in (None, "chat") else opted_in_group,
+                "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused", "tags": ["vision"]},
+                "model_info": {"id": "opted-in", "allow_fail_open": opted_in_group is not None, **team_info},
+            },
+            {
+                "model_name": model_name,
+                "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused", "tags": ["region:eu"]},
+                "model_info": {"id": "remaining", **team_info},
+            },
+        ],
+        routing_groups=(
+            [{"group_name": "public-chat", "models": ["chat"], "routing_strategy": "simple-shuffle"}]
+            if route == "routing_group"
+            else None
+        ),
+        model_group_alias={"public-chat": "chat"} if route == "alias" else {},
+        enable_tag_filtering=True,
+        optional_pre_call_checks=["responses_api_deployment_check"],
+    )
+
+
+def _response_id_for_remaining_deployment() -> str:
+    return ResponsesAPIRequestUtils._build_responses_api_response_id(
+        custom_llm_provider="openai", model_id="remaining", response_id="resp_previous"
+    )
+
+
+@pytest.mark.asyncio()
+@pytest.mark.parametrize("narrowing", ("affinity", "cooldown"))
+@pytest.mark.parametrize("tags", (("component:chat-search",), ("&vision",), ("!region:eu",)))
+@pytest.mark.parametrize("route", ("model", "routing_group", "wildcard", "team", "alias"))
+async def test_tag_fail_open_survives_deployment_narrowing(
+    narrowing: Literal["affinity", "cooldown"], tags: tuple[str, ...], route: str
+) -> None:
+    router: Final = _fail_open_router(route=route)
+    requested_model: Final = "chat" if route == "model" else "chat-test" if route == "wildcard" else "public-chat"
+    with patch("time.time", return_value=1_700_000_000.0):
+        if narrowing == "cooldown":
+            router.cooldown_cache.add_deployment_to_cooldown(
+                model_id="opted-in",
+                original_exception=RuntimeError("unavailable"),
+                exception_status=500,
+                cooldown_time=60,
+            )
+        candidates: Final = await router.async_get_healthy_deployments(
+            model=requested_model,
+            request_kwargs={
+                "litellm_metadata": {"tags": list(tags), "inherited_tags": [], "user_api_key_team_id": "team-a"},
+                **(
+                    {"previous_response_id": _response_id_for_remaining_deployment()} if narrowing == "affinity" else {}
+                ),
+            },
+        )
+    assert isinstance(candidates, (list, tuple))
+    assert tuple(deployment["model_info"]["id"] for deployment in candidates) == ("remaining",)
+
+
+@pytest.mark.asyncio()
+@pytest.mark.parametrize("route", ("routing_group", "wildcard", "team", "alias"))
+async def test_tag_fail_open_resolves_public_model_names(route: str) -> None:
+    router: Final = _fail_open_router(route=route)
+    candidates: Final = await router.async_get_healthy_deployments(
+        model="chat-test" if route == "wildcard" else "public-chat",
+        request_kwargs={
+            "litellm_metadata": {"tags": ["component:chat-search"], "user_api_key_team_id": "team-a"},
+        },
+    )
+    assert isinstance(candidates, (list, tuple))
+    assert {deployment["model_info"]["id"] for deployment in candidates} == {"opted-in", "remaining"}
+
+
+@pytest.mark.asyncio()
+@pytest.mark.parametrize("opted_in_group", (None, "other-group"))
+async def test_tag_fail_open_requires_opt_in_in_the_requested_group(opted_in_group: str | None) -> None:
+    router: Final = _fail_open_router(opted_in_group)
+    with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value):
+        await router.async_get_healthy_deployments(
+            model="chat",
+            request_kwargs={
+                "previous_response_id": _response_id_for_remaining_deployment(),
+                "litellm_metadata": {"tags": ["component:chat-search"], "inherited_tags": []},
+            },
+        )
+
+
+@pytest.mark.asyncio()
+async def test_tag_fail_open_does_not_use_another_teams_opt_in() -> None:
+    router: Final = litellm.Router(
+        model_list=[
+            {
+                "model_name": "chat",
+                "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused"},
+                "model_info": {
+                    "id": team_id,
+                    "team_id": team_id,
+                    "team_public_model_name": "public-chat",
+                    "allow_fail_open": team_id == "team-a",
+                },
+            }
+            for team_id in ("team-a", "team-b")
+        ],
+        enable_tag_filtering=True,
+    )
+    with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value):
+        await router.async_get_healthy_deployments(
+            model="public-chat",
+            request_kwargs={
+                "litellm_metadata": {"tags": ["component:chat-search"], "user_api_key_team_id": "team-b"},
+            },
+        )
+
+
+@pytest.mark.asyncio()
+@pytest.mark.parametrize("inherited_tag", ("&vision", "!region:eu"))
+async def test_tag_fail_open_preserves_inherited_policy_after_affinity(inherited_tag: str) -> None:
+    router: Final = _fail_open_router()
+    with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value):
+        await router.async_get_healthy_deployments(
+            model="chat",
+            request_kwargs={
+                "previous_response_id": _response_id_for_remaining_deployment(),
+                "litellm_metadata": {
+                    "tags": [inherited_tag, "component:chat-search"],
+                    "inherited_tags": [inherited_tag],
+                },
+            },
+        )
+
+
 @pytest.mark.asyncio()
 async def test_allow_fail_open_required_and_unmatched_falls_back_to_default_pool():
     router = litellm.Router(
@@ -3305,10 +3454,12 @@ async def _routed_to(router: litellm.Router, metadata: dict[str, object]) -> str
 
 
 async def _pool(router: litellm.Router, metadata: dict[str, object]) -> frozenset[str]:
+    request_deployments: Final = router.get_model_list(model_name="chat")
     deployments: Final = await get_deployments_for_tag(
         llm_router_instance=router,
         model="chat",
-        healthy_deployments=router.get_model_list(model_name="chat"),
+        healthy_deployments=request_deployments,
+        request_deployments=request_deployments,
         request_kwargs={"metadata": metadata},
     )
     return frozenset(deployment["model_info"]["id"] for deployment in deployments)