Merge 340c5eb12a into 527d882508

litellm/router_strategy/tag_based_routing.py

@@ -12,7 +12,15 @@ import re
 from collections.abc import Iterable, Mapping, Sequence
 from dataclasses import replace
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, Literal, Protocol, TypedDict, overload
+from typing import (
+    TYPE_CHECKING,
+    Final,
+    Literal,
+    Protocol,
+    TypedDict,
+    cast,  # noqa: TID251  # Router pool narrowing
+    overload,
+)
 
 from typing_extensions import ReadOnly
 
@@ -784,6 +792,71 @@ def get_tags_from_request_kwargs(
     return []
 
 
+def can_satisfy_confirmed_routing_tags(
+    ctx: "RoutingContext",
+    model: str,
+    request_kwargs: Mapping[str, object] | None = None,
+    metadata_variable_name: Literal["metadata", "litellm_metadata"] | None = None,
+) -> bool:
+    """
+    Whether attempting ``model`` can possibly succeed under the request's
+    ``tag_routing_prefix``-confirmed tags.
+
+    Returns False only when confirmed routing tags make success structurally
+    impossible for every deployment in the model group — so
+    ``run_async_fallback`` can skip that leg without raising, logging ERROR, or
+    counting a deployment failure. Returns True when the check does not apply
+    (no prefix, no confirmed tags, tag filtering off) or the group might still
+    serve the request.
+    """
+    settings: Final = ctx.settings()
+    routing_prefix: Final = settings.tag_routing_prefix or ""
+    if not routing_prefix or request_kwargs is None:
+        return True
+
+    request_tags: Final = get_tags_from_request_kwargs(request_kwargs, metadata_variable_name)
+    rewritten_tags, routing_confirmed = _strip_routing_prefix(request_tags, routing_prefix)
+    if not routing_confirmed:
+        return True
+
+    deployments: Final = cast(  # cast-ok: [LIT006] router deployment dicts are read through _DeploymentLike
+        _DeploymentPool, _all_deployments_or_fallback(ctx, model, ())
+    )
+    if not deployments:
+        return True
+
+    request_enable_tag_filtering: Final = request_kwargs.get("enable_tag_filtering")
+    chain_enable_tag_filtering: Final = _chain_tag_filtering_override(ctx, model, deployments)
+    chain_default: Final = (
+        chain_enable_tag_filtering if chain_enable_tag_filtering is not None else settings.enable_tag_filtering
+    )
+    if request_enable_tag_filtering is not True and chain_default is not True:
+        return True
+
+    required_tags, positive_tags, excluded_patterns = _split_tags(rewritten_tags)
+    excluded_set: Final = frozenset(excluded_patterns)
+    required_set: Final = frozenset(required_tags)
+    candidates: Final = _require_all_tags(_exclude_deployments(deployments, excluded_set), required_set)
+
+    confirmed_required: Final = required_set & routing_confirmed
+    confirmed_excluded: Final = excluded_set & routing_confirmed
+    if not candidates and (confirmed_required or confirmed_excluded):
+        return False
+
+    confirmed_positive: Final = tuple(tag for tag in positive_tags if tag in routing_confirmed)
+    if not confirmed_positive:
+        return True
+
+    if any(d.get("litellm_params", {}).get("tag_regex") for d in candidates):
+        return True
+
+    match_any: Final = settings.tag_filtering_match_any
+    return any(
+        is_valid_deployment_tag(d.get("litellm_params", {}).get("tags") or (), positive_tags, match_any)
+        for d in candidates
+    )
+
+
 _get_tags_from_request_kwargs = get_tags_from_request_kwargs
 
 

litellm/router_utils/fallback_event_handlers.py

@@ -5,7 +5,7 @@ from dataclasses import dataclass, field
 from datetime import datetime
 from enum import Enum
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any, Final, cast  # noqa: TID251  # narrows untyped **kwargs for the tag check
 
 import httpx
 from pydantic import TypeAdapter
@@ -742,6 +742,30 @@ async def _is_fallback_target_within_budget(
     return False
 
 
+def _is_fallback_target_tag_satisfiable(
+    ctx: "RoutingContext",
+    fallback_entry: str | Mapping[str, object],
+    kwargs: Mapping[str, object],
+) -> bool:
+    """
+    Skip a fallback leg whose model group cannot satisfy confirmed tag-routing
+    tags. Mirrors the authorized/budget pre-checks: structurally unsatisfiable
+    legs must not be attempted, raised, logged at ERROR, or counted as failures.
+    """
+    target: Final = _get_fallback_target_model_group(fallback_entry)
+    if target is None:
+        return True
+    from litellm.router_strategy.tag_based_routing import can_satisfy_confirmed_routing_tags
+
+    if can_satisfy_confirmed_routing_tags(ctx, target, kwargs):
+        return True
+    verbose_router_logger.info(
+        "Skipping fallback to model_group = %s: no deployment can satisfy confirmed tag routing tags",
+        mask_sensitive_structure(fallback_entry),
+    )
+    return False
+
+
 def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
     """
     True when a file, batch, or fine-tuning job operation names an id that only exists
@@ -856,6 +880,12 @@ async def run_async_fallback(
             continue
         if not await _is_fallback_target_within_budget(ctx, mg, original_model_group, kwargs, attempted=attempted):
             continue
+        if not _is_fallback_target_tag_satisfiable(
+            ctx,
+            mg,
+            cast(Mapping[str, object], kwargs),  # cast-ok: [LIT006] **kwargs is untyped here
+        ):
+            continue
         attempt_key = fallback_attempt_key(mg)
         if attempt_key is not None:
             if attempt_key in attempted:

tests/unit/router_utils/test_fallback_event_handlers.py

@@ -812,6 +812,291 @@ async def test_run_async_fallback_does_not_consult_access_check_for_same_model_g
     assert router.access_checks == []
 
 
+class TagRoutingFallbackRouter(RouterDouble):
+    """Records fallback attempts while exposing tag-routing deployment lookups."""
+
+    enable_tag_filtering = True
+    tag_routing_prefix = "route:"
+    tag_filtering_match_any = True
+
+    def __init__(self, deployments_by_group: dict[str, list[dict]]):
+        self.deployments_by_group = deployments_by_group
+        self.attempted_model_groups = []
+
+    def settings(self) -> RouterSettings:
+        return RouterSettings(
+            fallback_access_check=self.fallback_access_check,
+            fallback_budget_check=self.fallback_budget_check,
+            enable_tag_filtering=self.enable_tag_filtering,
+            tag_routing_prefix=self.tag_routing_prefix,
+            tag_filtering_match_any=self.tag_filtering_match_any,
+        )
+
+    def registry(self):
+        return self
+
+    def deployments(self, model_name, *, source):
+        return list(self.deployments_by_group.get(model_name, []))
+
+    async def async_function_with_fallbacks(self, *args, **kwargs):
+        self.attempted_model_groups.append(kwargs.get("model"))
+        return StreamingWrapper()
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_skips_legs_unsatisfiable_under_confirmed_tags(caplog):
+    """Confirmed tag_routing_prefix tags must skip structurally impossible legs at INFO,
+    not attempt them (which would raise/log ERROR and inflate failure metrics)."""
+    import logging
+
+    router = TagRoutingFallbackRouter(
+        {
+            "chain/2-vertex": [
+                {
+                    "model_name": "chain/2-vertex",
+                    "litellm_params": {"tags": ["inference:vertex", "default"]},
+                }
+            ],
+            "chain/3-bedrock": [
+                {
+                    "model_name": "chain/3-bedrock",
+                    "litellm_params": {"tags": ["inference:bedrock", "default"]},
+                }
+            ],
+        }
+    )
+
+    with caplog.at_level(logging.INFO, logger="LiteLLM Router"):
+        response = await run_async_fallback(
+            litellm_routing_ctx=RoutingContext(router),
+            fallback_model_group=["chain/2-vertex", "chain/3-bedrock"],
+            original_model_group="chain/1-openai",
+            original_exception=RuntimeError("primary failed"),
+            max_fallbacks=5,
+            fallback_depth=0,
+            model="chain/1-openai",
+            metadata={"tags": ["route:inference:bedrock"]},
+        )
+
+    assert router.attempted_model_groups == ["chain/3-bedrock"]
+    assert response._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1
+    skip_logs = [r for r in caplog.records if "no deployment can satisfy confirmed tag routing tags" in r.getMessage()]
+    assert len(skip_logs) == 1
+    assert skip_logs[0].levelno == logging.INFO
+    assert "chain/2-vertex" in skip_logs[0].getMessage()
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_raises_original_when_all_legs_tag_unsatisfiable():
+    router = TagRoutingFallbackRouter(
+        {
+            "chain/2-vertex": [
+                {
+                    "model_name": "chain/2-vertex",
+                    "litellm_params": {"tags": ["inference:vertex", "default"]},
+                }
+            ],
+            "chain/3-anthropic": [
+                {
+                    "model_name": "chain/3-anthropic",
+                    "litellm_params": {"tags": ["inference:anthropic", "default"]},
+                }
+            ],
+        }
+    )
+
+    with pytest.raises(RuntimeError, match="primary failed"):
+        await run_async_fallback(
+            litellm_routing_ctx=RoutingContext(router),
+            fallback_model_group=["chain/2-vertex", "chain/3-anthropic"],
+            original_model_group="chain/1-openai",
+            original_exception=RuntimeError("primary failed"),
+            max_fallbacks=5,
+            fallback_depth=0,
+            model="chain/1-openai",
+            metadata={"tags": ["route:inference:bedrock"]},
+        )
+
+    assert router.attempted_model_groups == []
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_does_not_skip_when_tag_routing_prefix_unset():
+    router = TagRoutingFallbackRouter(
+        {
+            "chain/2-vertex": [
+                {
+                    "model_name": "chain/2-vertex",
+                    "litellm_params": {"tags": ["inference:vertex", "default"]},
+                }
+            ],
+        }
+    )
+    router.tag_routing_prefix = ""
+
+    await run_async_fallback(
+        litellm_routing_ctx=RoutingContext(router),
+        fallback_model_group=["chain/2-vertex"],
+        original_model_group="chain/1-openai",
+        original_exception=RuntimeError("primary failed"),
+        max_fallbacks=3,
+        fallback_depth=0,
+        model="chain/1-openai",
+        metadata={"tags": ["route:inference:bedrock"]},
+    )
+
+    assert router.attempted_model_groups == ["chain/2-vertex"]
+
+
+def test_can_satisfy_confirmed_routing_tags_false_for_mismatched_single_deployment():
+    from litellm.router_strategy.tag_based_routing import can_satisfy_confirmed_routing_tags
+
+    router = TagRoutingFallbackRouter(
+        {
+            "chain/2-vertex": [
+                {
+                    "model_name": "chain/2-vertex",
+                    "litellm_params": {"tags": ["inference:vertex", "default"]},
+                }
+            ],
+        }
+    )
+    assert (
+        can_satisfy_confirmed_routing_tags(
+            RoutingContext(router),
+            "chain/2-vertex",
+            {"metadata": {"tags": ["route:inference:bedrock"]}},
+        )
+        is False
+    )
+
+
+def test_can_satisfy_confirmed_routing_tags_true_when_deployment_matches():
+    from litellm.router_strategy.tag_based_routing import can_satisfy_confirmed_routing_tags
+
+    router = TagRoutingFallbackRouter(
+        {
+            "chain/3-bedrock": [
+                {
+                    "model_name": "chain/3-bedrock",
+                    "litellm_params": {"tags": ["inference:bedrock", "default"]},
+                }
+            ],
+        }
+    )
+    assert (
+        can_satisfy_confirmed_routing_tags(
+            RoutingContext(router),
+            "chain/3-bedrock",
+            {"metadata": {"tags": ["route:inference:bedrock"]}},
+        )
+        is True
+    )
+
+
+_VERTEX_ONLY = {
+    "chain/2-vertex": [
+        {
+            "model_name": "chain/2-vertex",
+            "litellm_params": {"tags": ["inference:vertex", "default"]},
+        }
+    ],
+}
+_VERTEX_REGEX = {
+    "chain/2-vertex": [
+        {
+            "model_name": "chain/2-vertex",
+            "litellm_params": {"tags": ["inference:vertex"], "tag_regex": ["^inference:"]},
+        }
+    ],
+}
+
+
+@pytest.mark.parametrize(
+    ("deployments", "router_filtering", "model", "request_kwargs", "expected"),
+    [
+        pytest.param(_VERTEX_ONLY, True, "chain/2-vertex", None, True, id="no-request-kwargs"),
+        pytest.param(
+            _VERTEX_ONLY, True, "chain/2-vertex", {"metadata": {"tags": ["inference:bedrock"]}}, True, id="unconfirmed"
+        ),
+        pytest.param(
+            _VERTEX_ONLY,
+            True,
+            "chain/unknown",
+            {"metadata": {"tags": ["route:inference:bedrock"]}},
+            True,
+            id="no-group",
+        ),
+        pytest.param(
+            _VERTEX_ONLY,
+            False,
+            "chain/2-vertex",
+            {"metadata": {"tags": ["route:inference:bedrock"]}},
+            True,
+            id="filtering-off",
+        ),
+        pytest.param(
+            _VERTEX_ONLY,
+            False,
+            "chain/2-vertex",
+            {"metadata": {"tags": ["route:inference:bedrock"]}, "enable_tag_filtering": True},
+            False,
+            id="request-enables-filtering",
+        ),
+        pytest.param(
+            _VERTEX_ONLY,
+            True,
+            "chain/2-vertex",
+            {"metadata": {"tags": ["route:&inference:bedrock"]}},
+            False,
+            id="confirmed-required-missing",
+        ),
+        pytest.param(
+            _VERTEX_ONLY,
+            True,
+            "chain/2-vertex",
+            {"metadata": {"tags": ["route:!inference:vertex"]}},
+            False,
+            id="confirmed-excluded-all",
+        ),
+        pytest.param(
+            _VERTEX_ONLY,
+            True,
+            "chain/2-vertex",
+            {"metadata": {"tags": ["route:&inference:vertex"]}},
+            True,
+            id="confirmed-required-present",
+        ),
+        pytest.param(
+            _VERTEX_REGEX,
+            True,
+            "chain/2-vertex",
+            {"metadata": {"tags": ["route:inference:bedrock"]}},
+            True,
+            id="tag-regex-deployment",
+        ),
+    ],
+)
+def test_can_satisfy_confirmed_routing_tags_branches(deployments, router_filtering, model, request_kwargs, expected):
+    from litellm.router_strategy.tag_based_routing import can_satisfy_confirmed_routing_tags
+
+    router = TagRoutingFallbackRouter(deployments)
+    router.enable_tag_filtering = router_filtering
+    assert can_satisfy_confirmed_routing_tags(RoutingContext(router), model, request_kwargs) is expected
+
+
+def test_is_fallback_target_tag_satisfiable_allows_entry_without_model():
+    from litellm.router_utils.fallback_event_handlers import _is_fallback_target_tag_satisfiable
+
+    router = TagRoutingFallbackRouter(_VERTEX_ONLY)
+    assert (
+        _is_fallback_target_tag_satisfiable(
+            RoutingContext(router), {"messages": []}, {"metadata": {"tags": ["route:x"]}}
+        )
+        is True
+    )
+
+
 class RecordingFailRouter(RouterDouble):
     def __init__(self):
         self.attempted_models = []