diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index d3996f504a5..4c43fd4d86c 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -12,7 +12,16 @@ import re from collections.abc import Iterable, Mapping, Sequence from dataclasses import replace from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, overload +from typing import ( + TYPE_CHECKING, + Any, + Final, + Literal, + Protocol, + TypedDict, + cast, # noqa: TID251 # Router pool narrowing + overload, +) from typing_extensions import ReadOnly @@ -786,6 +795,72 @@ def get_tags_from_request_kwargs( return [] +def can_satisfy_confirmed_routing_tags( + llm_router_instance: LitellmRouter, + model: str, + request_kwargs: Mapping[str, object] | None = None, + metadata_variable_name: Literal["metadata", "litellm_metadata"] | None = None, +) -> bool: + """ + Whether attempting ``model`` can possibly succeed under the request's + ``tag_routing_prefix``-confirmed tags. + + Returns False only when confirmed routing tags make success structurally + impossible for every deployment in the model group — so + ``run_async_fallback`` can skip that leg without raising, logging ERROR, or + counting a deployment failure. Returns True when the check does not apply + (no prefix, no confirmed tags, tag filtering off) or the group might still + serve the request. + """ + routing_prefix: Final = llm_router_instance.tag_routing_prefix or "" + if not routing_prefix or request_kwargs is None: + return True + + request_tags: Final = get_tags_from_request_kwargs(request_kwargs, metadata_variable_name) + rewritten_tags, routing_confirmed = _strip_routing_prefix(request_tags, routing_prefix) + if not routing_confirmed: + return True + + deployments: Final = cast( # cast-ok: [LIT006] router deployment dicts are read through _DeploymentLike + _DeploymentPool, _all_deployments_or_fallback(llm_router_instance, model, ()) + ) + if not deployments: + return True + + request_enable_tag_filtering: Final = request_kwargs.get("enable_tag_filtering") + chain_enable_tag_filtering: Final = _chain_tag_filtering_override(llm_router_instance, model, deployments) + chain_default: Final = ( + chain_enable_tag_filtering + if chain_enable_tag_filtering is not None + else llm_router_instance.enable_tag_filtering + ) + if request_enable_tag_filtering is not True and chain_default is not True: + return True + + required_tags, positive_tags, excluded_patterns = _split_tags(rewritten_tags) + excluded_set: Final = frozenset(excluded_patterns) + required_set: Final = frozenset(required_tags) + candidates: Final = _require_all_tags(_exclude_deployments(deployments, excluded_set), required_set) + + confirmed_required: Final = required_set & routing_confirmed + confirmed_excluded: Final = excluded_set & routing_confirmed + if not candidates and (confirmed_required or confirmed_excluded): + return False + + confirmed_positive: Final = tuple(tag for tag in positive_tags if tag in routing_confirmed) + if not confirmed_positive: + return True + + if any(d.get("litellm_params", {}).get("tag_regex") for d in candidates): + return True + + match_any: Final = llm_router_instance.tag_filtering_match_any + return any( + is_valid_deployment_tag(d.get("litellm_params", {}).get("tags") or (), positive_tags, match_any) + for d in candidates + ) + + _get_tags_from_request_kwargs = get_tags_from_request_kwargs diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 6d1de1f9f6c..34f57f8b68d 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from datetime import datetime from enum import Enum from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final +from typing import TYPE_CHECKING, Any, Final, cast # noqa: TID251 # narrows untyped **kwargs for the tag check import httpx from pydantic import TypeAdapter @@ -745,6 +745,30 @@ async def _is_fallback_target_within_budget( return False +def _is_fallback_target_tag_satisfiable( + litellm_router: LitellmRouter, + fallback_entry: str | Mapping[str, object], + kwargs: Mapping[str, object], +) -> bool: + """ + Skip a fallback leg whose model group cannot satisfy confirmed tag-routing + tags. Mirrors the authorized/budget pre-checks: structurally unsatisfiable + legs must not be attempted, raised, logged at ERROR, or counted as failures. + """ + target: Final = _get_fallback_target_model_group(fallback_entry) + if target is None: + return True + from litellm.router_strategy.tag_based_routing import can_satisfy_confirmed_routing_tags + + if can_satisfy_confirmed_routing_tags(litellm_router, target, kwargs): + return True + verbose_router_logger.info( + "Skipping fallback to model_group = %s: no deployment can satisfy confirmed tag routing tags", + mask_sensitive_structure(fallback_entry), + ) + return False + + def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool: """ True when a file, batch, or fine-tuning job operation names an id that only exists @@ -861,6 +885,12 @@ async def run_async_fallback( litellm_router, mg, original_model_group, kwargs, attempted=attempted ): continue + if not _is_fallback_target_tag_satisfiable( + litellm_router, + mg, + cast(Mapping[str, object], kwargs), # cast-ok: [LIT006] **kwargs is untyped here + ): + continue attempt_key = fallback_attempt_key(mg) if attempt_key is not None: if attempt_key in attempted: diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index c5a476c450b..6b4cdf37a71 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -61,6 +61,7 @@ class StreamingWrapper: class FakeRouter: fallback_access_check = None fallback_budget_check = None + tag_routing_prefix = None def log_retry(self, kwargs, e): return kwargs @@ -72,6 +73,7 @@ class FakeRouter: class AlwaysFailRouter: fallback_access_check = None fallback_budget_check = None + tag_routing_prefix = None def log_retry(self, kwargs, e): return kwargs @@ -137,6 +139,7 @@ async def test_run_async_fallback_raises_when_all_fallbacks_fail(): class RecordingRouter: fallback_access_check = None fallback_budget_check = None + tag_routing_prefix = None def __init__(self): self.received_kwargs = None @@ -413,6 +416,7 @@ async def test_run_async_fallback_skips_original_model_group(): class AttemptRecordingRouter: fallback_access_check = None fallback_budget_check = None + tag_routing_prefix = None def __init__(self): self.attempted_model_groups = [] @@ -793,9 +797,283 @@ async def test_run_async_fallback_does_not_consult_access_check_for_same_model_g assert router.access_checks == [] +class TagRoutingFallbackRouter: + """Records fallback attempts while exposing tag-routing deployment lookups.""" + + fallback_access_check = None + fallback_budget_check = None + enable_tag_filtering = True + tag_routing_prefix = "route:" + tag_filtering_match_any = True + + def __init__(self, deployments_by_group: dict[str, list[dict]]): + self.deployments_by_group = deployments_by_group + self.attempted_model_groups = [] + + def log_retry(self, kwargs, e): + return kwargs + + def get_all_deployments(self, model_name: str, **kwargs): + return list(self.deployments_by_group.get(model_name, [])) + + async def async_function_with_fallbacks(self, *args, **kwargs): + self.attempted_model_groups.append(kwargs.get("model")) + return StreamingWrapper() + + +@pytest.mark.asyncio +async def test_run_async_fallback_skips_legs_unsatisfiable_under_confirmed_tags(caplog): + """Confirmed tag_routing_prefix tags must skip structurally impossible legs at INFO, + not attempt them (which would raise/log ERROR and inflate failure metrics).""" + import logging + + router = TagRoutingFallbackRouter( + { + "chain/2-vertex": [ + { + "model_name": "chain/2-vertex", + "litellm_params": {"tags": ["inference:vertex", "default"]}, + } + ], + "chain/3-bedrock": [ + { + "model_name": "chain/3-bedrock", + "litellm_params": {"tags": ["inference:bedrock", "default"]}, + } + ], + } + ) + + with caplog.at_level(logging.INFO, logger="LiteLLM Router"): + response = await run_async_fallback( + litellm_router=router, + fallback_model_group=["chain/2-vertex", "chain/3-bedrock"], + original_model_group="chain/1-openai", + original_exception=RuntimeError("primary failed"), + max_fallbacks=5, + fallback_depth=0, + model="chain/1-openai", + metadata={"tags": ["route:inference:bedrock"]}, + ) + + assert router.attempted_model_groups == ["chain/3-bedrock"] + assert response._hidden_params["additional_headers"]["x-litellm-attempted-fallbacks"] == 1 + skip_logs = [r for r in caplog.records if "no deployment can satisfy confirmed tag routing tags" in r.getMessage()] + assert len(skip_logs) == 1 + assert skip_logs[0].levelno == logging.INFO + assert "chain/2-vertex" in skip_logs[0].getMessage() + + +@pytest.mark.asyncio +async def test_run_async_fallback_raises_original_when_all_legs_tag_unsatisfiable(): + router = TagRoutingFallbackRouter( + { + "chain/2-vertex": [ + { + "model_name": "chain/2-vertex", + "litellm_params": {"tags": ["inference:vertex", "default"]}, + } + ], + "chain/3-anthropic": [ + { + "model_name": "chain/3-anthropic", + "litellm_params": {"tags": ["inference:anthropic", "default"]}, + } + ], + } + ) + + with pytest.raises(RuntimeError, match="primary failed"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=["chain/2-vertex", "chain/3-anthropic"], + original_model_group="chain/1-openai", + original_exception=RuntimeError("primary failed"), + max_fallbacks=5, + fallback_depth=0, + model="chain/1-openai", + metadata={"tags": ["route:inference:bedrock"]}, + ) + + assert router.attempted_model_groups == [] + + +@pytest.mark.asyncio +async def test_run_async_fallback_does_not_skip_when_tag_routing_prefix_unset(): + router = TagRoutingFallbackRouter( + { + "chain/2-vertex": [ + { + "model_name": "chain/2-vertex", + "litellm_params": {"tags": ["inference:vertex", "default"]}, + } + ], + } + ) + router.tag_routing_prefix = "" + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["chain/2-vertex"], + original_model_group="chain/1-openai", + original_exception=RuntimeError("primary failed"), + max_fallbacks=3, + fallback_depth=0, + model="chain/1-openai", + metadata={"tags": ["route:inference:bedrock"]}, + ) + + assert router.attempted_model_groups == ["chain/2-vertex"] + + +def test_can_satisfy_confirmed_routing_tags_false_for_mismatched_single_deployment(): + from litellm.router_strategy.tag_based_routing import can_satisfy_confirmed_routing_tags + + router = TagRoutingFallbackRouter( + { + "chain/2-vertex": [ + { + "model_name": "chain/2-vertex", + "litellm_params": {"tags": ["inference:vertex", "default"]}, + } + ], + } + ) + assert ( + can_satisfy_confirmed_routing_tags( + router, + "chain/2-vertex", + {"metadata": {"tags": ["route:inference:bedrock"]}}, + ) + is False + ) + + +def test_can_satisfy_confirmed_routing_tags_true_when_deployment_matches(): + from litellm.router_strategy.tag_based_routing import can_satisfy_confirmed_routing_tags + + router = TagRoutingFallbackRouter( + { + "chain/3-bedrock": [ + { + "model_name": "chain/3-bedrock", + "litellm_params": {"tags": ["inference:bedrock", "default"]}, + } + ], + } + ) + assert ( + can_satisfy_confirmed_routing_tags( + router, + "chain/3-bedrock", + {"metadata": {"tags": ["route:inference:bedrock"]}}, + ) + is True + ) + + +_VERTEX_ONLY = { + "chain/2-vertex": [ + { + "model_name": "chain/2-vertex", + "litellm_params": {"tags": ["inference:vertex", "default"]}, + } + ], +} +_VERTEX_REGEX = { + "chain/2-vertex": [ + { + "model_name": "chain/2-vertex", + "litellm_params": {"tags": ["inference:vertex"], "tag_regex": ["^inference:"]}, + } + ], +} + + +@pytest.mark.parametrize( + ("deployments", "router_filtering", "model", "request_kwargs", "expected"), + [ + pytest.param(_VERTEX_ONLY, True, "chain/2-vertex", None, True, id="no-request-kwargs"), + pytest.param( + _VERTEX_ONLY, True, "chain/2-vertex", {"metadata": {"tags": ["inference:bedrock"]}}, True, id="unconfirmed" + ), + pytest.param( + _VERTEX_ONLY, + True, + "chain/unknown", + {"metadata": {"tags": ["route:inference:bedrock"]}}, + True, + id="no-group", + ), + pytest.param( + _VERTEX_ONLY, + False, + "chain/2-vertex", + {"metadata": {"tags": ["route:inference:bedrock"]}}, + True, + id="filtering-off", + ), + pytest.param( + _VERTEX_ONLY, + False, + "chain/2-vertex", + {"metadata": {"tags": ["route:inference:bedrock"]}, "enable_tag_filtering": True}, + False, + id="request-enables-filtering", + ), + pytest.param( + _VERTEX_ONLY, + True, + "chain/2-vertex", + {"metadata": {"tags": ["route:&inference:bedrock"]}}, + False, + id="confirmed-required-missing", + ), + pytest.param( + _VERTEX_ONLY, + True, + "chain/2-vertex", + {"metadata": {"tags": ["route:!inference:vertex"]}}, + False, + id="confirmed-excluded-all", + ), + pytest.param( + _VERTEX_ONLY, + True, + "chain/2-vertex", + {"metadata": {"tags": ["route:&inference:vertex"]}}, + True, + id="confirmed-required-present", + ), + pytest.param( + _VERTEX_REGEX, + True, + "chain/2-vertex", + {"metadata": {"tags": ["route:inference:bedrock"]}}, + True, + id="tag-regex-deployment", + ), + ], +) +def test_can_satisfy_confirmed_routing_tags_branches(deployments, router_filtering, model, request_kwargs, expected): + from litellm.router_strategy.tag_based_routing import can_satisfy_confirmed_routing_tags + + router = TagRoutingFallbackRouter(deployments) + router.enable_tag_filtering = router_filtering + assert can_satisfy_confirmed_routing_tags(router, model, request_kwargs) is expected + + +def test_is_fallback_target_tag_satisfiable_allows_entry_without_model(): + from litellm.router_utils.fallback_event_handlers import _is_fallback_target_tag_satisfiable + + router = TagRoutingFallbackRouter(_VERTEX_ONLY) + assert _is_fallback_target_tag_satisfiable(router, {"messages": []}, {"metadata": {"tags": ["route:x"]}}) is True + + class RecordingFailRouter: fallback_access_check = None fallback_budget_check = None + tag_routing_prefix = None def __init__(self): self.attempted_models = [] @@ -1378,6 +1656,7 @@ class TestRunAsyncFallbackTriggersCooldown: class RouterWithLoggingKwarg: fallback_access_check = None fallback_budget_check = None + tag_routing_prefix = None def __init__(self): self.cooldown_time = 60.0