diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index f2aafa5b7df..bf6d9b4aa41 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -12,7 +12,15 @@ import re from collections.abc import Iterable, Mapping, Sequence from dataclasses import replace from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Literal, Protocol, TypedDict, overload +from typing import ( + TYPE_CHECKING, + Final, + Literal, + Protocol, + TypedDict, + cast, # noqa: TID251 # Router pool narrowing + overload, +) from typing_extensions import ReadOnly @@ -784,6 +792,71 @@ def get_tags_from_request_kwargs( return [] +def can_satisfy_confirmed_routing_tags( + ctx: "RoutingContext", + model: str, + request_kwargs: Mapping[str, object] | None = None, + metadata_variable_name: Literal["metadata", "litellm_metadata"] | None = None, +) -> bool: + """ + Whether attempting ``model`` can possibly succeed under the request's + ``tag_routing_prefix``-confirmed tags. + + Returns False only when confirmed routing tags make success structurally + impossible for every deployment in the model group — so + ``run_async_fallback`` can skip that leg without raising, logging ERROR, or + counting a deployment failure. Returns True when the check does not apply + (no prefix, no confirmed tags, tag filtering off) or the group might still + serve the request. + """ + settings: Final = ctx.settings() + routing_prefix: Final = settings.tag_routing_prefix or "" + if not routing_prefix or request_kwargs is None: + return True + + request_tags: Final = get_tags_from_request_kwargs(request_kwargs, metadata_variable_name) + rewritten_tags, routing_confirmed = _strip_routing_prefix(request_tags, routing_prefix) + if not routing_confirmed: + return True + + deployments: Final = cast( # cast-ok: [LIT006] router deployment dicts are read through _DeploymentLike + _DeploymentPool, _all_deployments_or_fallback(ctx, model, ()) + ) + if not deployments: + return True + + request_enable_tag_filtering: Final = request_kwargs.get("enable_tag_filtering") + chain_enable_tag_filtering: Final = _chain_tag_filtering_override(ctx, model, deployments) + chain_default: Final = ( + chain_enable_tag_filtering if chain_enable_tag_filtering is not None else settings.enable_tag_filtering + ) + if request_enable_tag_filtering is not True and chain_default is not True: + return True + + required_tags, positive_tags, excluded_patterns = _split_tags(rewritten_tags) + excluded_set: Final = frozenset(excluded_patterns) + required_set: Final = frozenset(required_tags) + candidates: Final = _require_all_tags(_exclude_deployments(deployments, excluded_set), required_set) + + confirmed_required: Final = required_set & routing_confirmed + confirmed_excluded: Final = excluded_set & routing_confirmed + if not candidates and (confirmed_required or confirmed_excluded): + return False + + confirmed_positive: Final = tuple(tag for tag in positive_tags if tag in routing_confirmed) + if not confirmed_positive: + return True + + if any(d.get("litellm_params", {}).get("tag_regex") for d in candidates): + return True + + match_any: Final = settings.tag_filtering_match_any + return any( + is_valid_deployment_tag(d.get("litellm_params", {}).get("tags") or (), positive_tags, match_any) + for d in candidates + ) + + _get_tags_from_request_kwargs = get_tags_from_request_kwargs diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index c929cca44be..6b3bfc7f9ef 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from datetime import datetime from enum import Enum from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast # noqa: TID251 # narrows untyped **kwargs for the tag check import httpx from pydantic import TypeAdapter @@ -742,6 +742,30 @@ async def _is_fallback_target_within_budget( return False +def _is_fallback_target_tag_satisfiable( + ctx: "RoutingContext", + fallback_entry: str | Mapping[str, object], + kwargs: Mapping[str, object], +) -> bool: + """ + Skip a fallback leg whose model group cannot satisfy confirmed tag-routing + tags. Mirrors the authorized/budget pre-checks: structurally unsatisfiable + legs must not be attempted, raised, logged at ERROR, or counted as failures. + """ + target: Final = _get_fallback_target_model_group(fallback_entry) + if target is None: + return True + from litellm.router_strategy.tag_based_routing import can_satisfy_confirmed_routing_tags + + if can_satisfy_confirmed_routing_tags(ctx, target, kwargs): + return True + verbose_router_logger.info( + "Skipping fallback to model_group = %s: no deployment can satisfy confirmed tag routing tags", + mask_sensitive_structure(fallback_entry), + ) + return False + + def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool: """ True when a file, batch, or fine-tuning job operation names an id that only exists @@ -856,6 +880,12 @@ async def run_async_fallback( continue if not await _is_fallback_target_within_budget(ctx, mg, original_model_group, kwargs, attempted=attempted): continue + if not _is_fallback_target_tag_satisfiable( + ctx, + mg, + cast(Mapping[str, object], kwargs), # cast-ok: [LIT006] **kwargs is untyped here + ): + continue attempt_key = fallback_attempt_key(mg) if attempt_key is not None: if attempt_key in attempted: diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index 26dbb96c60a..e739a9b1913 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -812,6 +812,291 @@ async def test_run_async_fallback_does_not_consult_access_check_for_same_model_g assert router.access_checks == [] +class TagRoutingFallbackRouter(RouterDouble): + """Records fallback attempts while exposing tag-routing deployment lookups.""" + + enable_tag_filtering = True + tag_routing_prefix = "route:" + tag_filtering_match_any = True + + def __init__(self, deployments_by_group: dict[str, list[dict]]): + self.deployments_by_group = deployments_by_group + self.attempted_model_groups = [] + + def settings(self) -> RouterSettings: + return RouterSettings( + fallback_access_check=self.fallback_access_check, + fallback_budget_check=self.fallback_budget_check, + enable_tag_filtering=self.enable_tag_filtering, + tag_routing_prefix=self.tag_routing_prefix, + tag_filtering_match_any=self.tag_filtering_match_any, + ) + + def registry(self): + return self + + def deployments(self, model_name, *, source): + return list(self.deployments_by_group.get(model_name, [])) + + async def async_function_with_fallbacks(self, *args, **kwargs): + self.attempted_model_groups.append(kwargs.get("model")) + return StreamingWrapper() + + +@pytest.mark.asyncio +async def test_run_async_fallback_skips_legs_unsatisfiable_under_confirmed_tags(caplog): + """Confirmed tag_routing_prefix tags must skip structurally impossible legs at INFO, + not attempt them (which would raise/log ERROR and inflate failure metrics).""" + import logging + + router = TagRoutingFallbackRouter( + { + "chain/2-vertex": [ + { + "model_name": "chain/2-vertex", + "litellm_params": {"tags": ["inference:vertex", "default"]}, + } + ], + "chain/3-bedrock": [ + { + "model_name": "chain/3-bedrock", + "litellm_params": {"tags": ["inference:bedrock", "default"]}, + } + ], + } + ) + + with caplog.at_level(logging.INFO, logger="LiteLLM Router"): + response = await run_async_fallback( + litellm_routing_ctx=RoutingContext(router), + fallback_model_group=["chain/2-vertex", "chain/3-bedrock"], + original_model_group="chain/1-openai", + original_exception=RuntimeError("primary failed"), + max_fallbacks=5, + fallback_depth=0, + model="chain/1-openai", + metadata={"tags": ["route:inference:bedrock"]}, + ) + + assert router.attempted_model_groups == ["chain/3-bedrock"] + assert response._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1 + skip_logs = [r for r in caplog.records if "no deployment can satisfy confirmed tag routing tags" in r.getMessage()] + assert len(skip_logs) == 1 + assert skip_logs[0].levelno == logging.INFO + assert "chain/2-vertex" in skip_logs[0].getMessage() + + +@pytest.mark.asyncio +async def test_run_async_fallback_raises_original_when_all_legs_tag_unsatisfiable(): + router = TagRoutingFallbackRouter( + { + "chain/2-vertex": [ + { + "model_name": "chain/2-vertex", + "litellm_params": {"tags": ["inference:vertex", "default"]}, + } + ], + "chain/3-anthropic": [ + { + "model_name": "chain/3-anthropic", + "litellm_params": {"tags": ["inference:anthropic", "default"]}, + } + ], + } + ) + + with pytest.raises(RuntimeError, match="primary failed"): + await run_async_fallback( + litellm_routing_ctx=RoutingContext(router), + fallback_model_group=["chain/2-vertex", "chain/3-anthropic"], + original_model_group="chain/1-openai", + original_exception=RuntimeError("primary failed"), + max_fallbacks=5, + fallback_depth=0, + model="chain/1-openai", + metadata={"tags": ["route:inference:bedrock"]}, + ) + + assert router.attempted_model_groups == [] + + +@pytest.mark.asyncio +async def test_run_async_fallback_does_not_skip_when_tag_routing_prefix_unset(): + router = TagRoutingFallbackRouter( + { + "chain/2-vertex": [ + { + "model_name": "chain/2-vertex", + "litellm_params": {"tags": ["inference:vertex", "default"]}, + } + ], + } + ) + router.tag_routing_prefix = "" + + await run_async_fallback( + litellm_routing_ctx=RoutingContext(router), + fallback_model_group=["chain/2-vertex"], + original_model_group="chain/1-openai", + original_exception=RuntimeError("primary failed"), + max_fallbacks=3, + fallback_depth=0, + model="chain/1-openai", + metadata={"tags": ["route:inference:bedrock"]}, + ) + + assert router.attempted_model_groups == ["chain/2-vertex"] + + +def test_can_satisfy_confirmed_routing_tags_false_for_mismatched_single_deployment(): + from litellm.router_strategy.tag_based_routing import can_satisfy_confirmed_routing_tags + + router = TagRoutingFallbackRouter( + { + "chain/2-vertex": [ + { + "model_name": "chain/2-vertex", + "litellm_params": {"tags": ["inference:vertex", "default"]}, + } + ], + } + ) + assert ( + can_satisfy_confirmed_routing_tags( + RoutingContext(router), + "chain/2-vertex", + {"metadata": {"tags": ["route:inference:bedrock"]}}, + ) + is False + ) + + +def test_can_satisfy_confirmed_routing_tags_true_when_deployment_matches(): + from litellm.router_strategy.tag_based_routing import can_satisfy_confirmed_routing_tags + + router = TagRoutingFallbackRouter( + { + "chain/3-bedrock": [ + { + "model_name": "chain/3-bedrock", + "litellm_params": {"tags": ["inference:bedrock", "default"]}, + } + ], + } + ) + assert ( + can_satisfy_confirmed_routing_tags( + RoutingContext(router), + "chain/3-bedrock", + {"metadata": {"tags": ["route:inference:bedrock"]}}, + ) + is True + ) + + +_VERTEX_ONLY = { + "chain/2-vertex": [ + { + "model_name": "chain/2-vertex", + "litellm_params": {"tags": ["inference:vertex", "default"]}, + } + ], +} +_VERTEX_REGEX = { + "chain/2-vertex": [ + { + "model_name": "chain/2-vertex", + "litellm_params": {"tags": ["inference:vertex"], "tag_regex": ["^inference:"]}, + } + ], +} + + +@pytest.mark.parametrize( + ("deployments", "router_filtering", "model", "request_kwargs", "expected"), + [ + pytest.param(_VERTEX_ONLY, True, "chain/2-vertex", None, True, id="no-request-kwargs"), + pytest.param( + _VERTEX_ONLY, True, "chain/2-vertex", {"metadata": {"tags": ["inference:bedrock"]}}, True, id="unconfirmed" + ), + pytest.param( + _VERTEX_ONLY, + True, + "chain/unknown", + {"metadata": {"tags": ["route:inference:bedrock"]}}, + True, + id="no-group", + ), + pytest.param( + _VERTEX_ONLY, + False, + "chain/2-vertex", + {"metadata": {"tags": ["route:inference:bedrock"]}}, + True, + id="filtering-off", + ), + pytest.param( + _VERTEX_ONLY, + False, + "chain/2-vertex", + {"metadata": {"tags": ["route:inference:bedrock"]}, "enable_tag_filtering": True}, + False, + id="request-enables-filtering", + ), + pytest.param( + _VERTEX_ONLY, + True, + "chain/2-vertex", + {"metadata": {"tags": ["route:&inference:bedrock"]}}, + False, + id="confirmed-required-missing", + ), + pytest.param( + _VERTEX_ONLY, + True, + "chain/2-vertex", + {"metadata": {"tags": ["route:!inference:vertex"]}}, + False, + id="confirmed-excluded-all", + ), + pytest.param( + _VERTEX_ONLY, + True, + "chain/2-vertex", + {"metadata": {"tags": ["route:&inference:vertex"]}}, + True, + id="confirmed-required-present", + ), + pytest.param( + _VERTEX_REGEX, + True, + "chain/2-vertex", + {"metadata": {"tags": ["route:inference:bedrock"]}}, + True, + id="tag-regex-deployment", + ), + ], +) +def test_can_satisfy_confirmed_routing_tags_branches(deployments, router_filtering, model, request_kwargs, expected): + from litellm.router_strategy.tag_based_routing import can_satisfy_confirmed_routing_tags + + router = TagRoutingFallbackRouter(deployments) + router.enable_tag_filtering = router_filtering + assert can_satisfy_confirmed_routing_tags(RoutingContext(router), model, request_kwargs) is expected + + +def test_is_fallback_target_tag_satisfiable_allows_entry_without_model(): + from litellm.router_utils.fallback_event_handlers import _is_fallback_target_tag_satisfiable + + router = TagRoutingFallbackRouter(_VERTEX_ONLY) + assert ( + _is_fallback_target_tag_satisfiable( + RoutingContext(router), {"messages": []}, {"metadata": {"tags": ["route:x"]}} + ) + is True + ) + + class RecordingFailRouter(RouterDouble): def __init__(self): self.attempted_models = []