scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-13 08:52:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 175

Merge pull request #50297 from BerriAI/litellm_backport_48470_lts_1_108_x

Browse source 
fix(router): backport #48470 to lts/1.108.x
This commit is contained in:
shrey 2026-11-05 20:59:32 -08:00 • committed by GitHub
commit bf06bf1edb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 189 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/router.py
View file

@ -14072,6 +14072,7 @@ class Router:
request_kwargs=request_kwargs,
healthy_deployments=healthy_deployments,
metadata_variable_name=self._get_metadata_variable_name_from_kwargs(request_kwargs),
request_deployments=request_deployments,
)
# narrow to whatever `self.routing_plugins` left in candidate_models

23
litellm/router_strategy/tag_based_routing.py
View file

@ -8,9 +8,11 @@ Use this to route requests between Teams
"""
import re
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypedDict, overload
from typing_extensions import ReadOnly
from litellm._logging import verbose_logger
from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY
@ -50,6 +52,10 @@ class _DeploymentLike(Protocol):
def get(self, key: Literal["model_name"], /) -> object: ...
class _DeploymentWithModelInfo(TypedDict, total=False):
model_info: ReadOnly[Mapping[str, object]]
class _MetadataLike(Protocol):
@overload
def get(self, key: Literal["tags"], /) -> Sequence[str] | None: ...
@ -279,13 +285,14 @@ def _unknown_required_tag_hides_an_answer(
def _chain_allows_fail_open(
healthy_deployments: _DeploymentPool,
request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo],
excluded_set: frozenset[str],
required_set: frozenset[str],
routing_confirmed: frozenset[str],
) -> bool:
if _unknown_required_tag_hides_an_answer(healthy_deployments, excluded_set, required_set, routing_confirmed):
return False
return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in healthy_deployments)
return any((d.get("model_info") or {}).get("allow_fail_open") is True for d in request_deployments)
def _trusted_only_pool(
@ -321,6 +328,7 @@ def _trusted_only_pool(
def _resolve_or_fail_open(
request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo],
pool: Sequence[_DeploymentLike],
healthy_deployments: _DeploymentPool,
excluded_set: frozenset[str],
@ -333,7 +341,7 @@ def _resolve_or_fail_open(
) -> tuple[_DeploymentLike, ...]:
if pool:
return tuple(pool)
if _chain_allows_fail_open(healthy_deployments, excluded_set, required_set, routing_confirmed):
if _chain_allows_fail_open(healthy_deployments, request_deployments, excluded_set, required_set, routing_confirmed):
# Fall open only within whatever still satisfies whichever constraints
# trace back to key/team policy. A constraint with no inherited backing at
# all (or, when inherited_tags is unavailable, any constraint at all) can
@ -350,6 +358,7 @@ def _resolve_or_fail_open(
def _resolve_constraint_only_pool(
request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo],
healthy_deployments: _DeploymentPool,
excluded_set: frozenset[str],
required_set: frozenset[str],
@ -365,6 +374,7 @@ def _resolve_constraint_only_pool(
else _exclude_deployments(_default_tagged_pool(healthy_deployments), excluded_set)
)
return _resolve_or_fail_open(
request_deployments,
pool,
healthy_deployments,
excluded_set,
@ -480,6 +490,8 @@ async def get_deployments_for_tag(
healthy_deployments: _DeploymentPool,
request_kwargs: _RequestKwargsLike | None = None,
metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata",
*,
request_deployments: Iterable[_DeploymentLike | _DeploymentWithModelInfo],
) -> _DeploymentPool:
"""
Returns a list of deployments that match the requested model and tags in the request.
@ -548,6 +560,7 @@ async def get_deployments_for_tag(
if constraint_only:
return _resolve_constraint_only_pool(
request_deployments,
healthy_deployments,
excluded_set,
required_set,
@ -599,6 +612,7 @@ async def get_deployments_for_tag(
if len(new_healthy_deployments) == 0 and len(default_deployments) == 0:
return _resolve_or_fail_open(
request_deployments,
(),
healthy_deployments,
excluded_set,
@ -616,6 +630,7 @@ async def get_deployments_for_tag(
and _tag_known_to_group(llm_router_instance, model, positive_tags, routing_confirmed)
):
return _resolve_or_fail_open(
request_deployments,
(),
healthy_deployments,
excluded_set,

11
tests/unit/router_strategy/test_router_tag_regex_routing.py
View file

@ -121,6 +121,7 @@ async def test_claude_code_ua_routes_to_cc_deployment():
llm_router_instance=router,
model="claude-sonnet",
healthy_deployments=ALL_DEPLOYMENTS,
request_deployments=ALL_DEPLOYMENTS,
request_kwargs={"metadata": {"user_agent": "claude-code/1.2.3"}},
)
assert len(result) == 1
@ -135,6 +136,7 @@ async def test_regular_ua_routes_to_default_deployment():
llm_router_instance=router,
model="claude-sonnet",
healthy_deployments=ALL_DEPLOYMENTS,
request_deployments=ALL_DEPLOYMENTS,
request_kwargs={"metadata": {"user_agent": "Mozilla/5.0 (browser)"}},
)
assert len(result) == 1
@ -149,6 +151,7 @@ async def test_no_ua_routes_to_default_deployment():
llm_router_instance=router,
model="claude-sonnet",
healthy_deployments=ALL_DEPLOYMENTS,
request_deployments=ALL_DEPLOYMENTS,
request_kwargs={"metadata": {}},
)
assert len(result) == 1
@ -164,6 +167,7 @@ async def test_tag_routing_metadata_written_for_regex_match():
llm_router_instance=router,
model="claude-sonnet",
healthy_deployments=ALL_DEPLOYMENTS,
request_deployments=ALL_DEPLOYMENTS,
request_kwargs={"metadata": metadata},
)
assert "tag_routing" in metadata
@ -181,6 +185,7 @@ async def test_tag_filtering_disabled_returns_all_deployments():
llm_router_instance=router,
model="claude-sonnet",
healthy_deployments=ALL_DEPLOYMENTS,
request_deployments=ALL_DEPLOYMENTS,
request_kwargs={"metadata": {"user_agent": "claude-code/1.0"}},
)
assert result == ALL_DEPLOYMENTS
@ -199,6 +204,7 @@ async def test_empty_healthy_deployments_with_request_tags_returns_empty_list():
llm_router_instance=router,
model="gpt-5.2",
healthy_deployments=[],
request_deployments=[],
request_kwargs={
"metadata": {"tags": ["client_id:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]}
},
@ -228,6 +234,7 @@ async def test_explicit_tag_match_takes_precedence_over_regex():
llm_router_instance=router,
model="claude-sonnet",
healthy_deployments=[deployment_with_both],
request_deployments=[deployment_with_both],
request_kwargs={"metadata": metadata},
)
assert len(result) == 1
@ -269,6 +276,7 @@ async def test_user_agent_present_no_tag_regex_deployments_does_not_raise():
llm_router_instance=router,
model="gpt-4",
healthy_deployments=plain_tag_only_deployments,
request_deployments=plain_tag_only_deployments,
request_kwargs={"metadata": {"user_agent": "Mozilla/5.0 (any-client)"}},
)
# Falls through to "return healthy_deployments" path unchanged
@ -305,6 +313,7 @@ async def test_tag_routing_metadata_not_overwritten_for_multiple_matches():
llm_router_instance=router,
model="claude-sonnet",
healthy_deployments=[deployment_a, deployment_b],
request_deployments=[deployment_a, deployment_b],
request_kwargs={"metadata": metadata},
)
assert len(result) == 2
@ -357,6 +366,7 @@ async def test_match_any_false_strict_tag_check_blocks_regex_fallback():
llm_router_instance=router,
model="claude-sonnet",
healthy_deployments=[deployment_strict, default_deployment],
request_deployments=[deployment_strict, default_deployment],
request_kwargs={"metadata": metadata},
)
ids = [d["model_info"]["id"] for d in result]
@ -387,6 +397,7 @@ async def test_match_any_false_regex_only_deployment_still_matches():
llm_router_instance=router,
model="claude-sonnet",
healthy_deployments=[regex_only_deployment],
request_deployments=[regex_only_deployment],
request_kwargs={"metadata": {"user_agent": "claude-code/1.0"}},
)
assert len(result) == 1

166
tests/unit/router_strategy/test_router_tag_routing.py
View file

@ -3,12 +3,15 @@
import logging
from typing import Final
from typing import Final, Literal
from unittest.mock import patch
import pytest
import litellm
from litellm._logging import verbose_logger
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.router import RouterErrors
async def _routed_model_ids(
@ -1365,14 +1368,17 @@ def test_chain_allows_fail_open_true_when_any_member_sets_flag():
{"model_info": {}, "litellm_params": {"tags": ["provider:anthropic"]}},
{"model_info": {"allow_fail_open": True}, "litellm_params": {"tags": ["provider:openai"]}},
]
assert _chain_allows_fail_open(deployments, frozenset(), frozenset({"provider:anthropic"}), frozenset()) is True
assert (
_chain_allows_fail_open(deployments, deployments, frozenset(), frozenset({"provider:anthropic"}), frozenset())
is True
)
def test_chain_allows_fail_open_false_by_default():
from litellm.router_strategy.tag_based_routing import _chain_allows_fail_open
deployments = [{"model_info": {}}, {}]
assert _chain_allows_fail_open(deployments, frozenset(), frozenset(), frozenset()) is False
assert _chain_allows_fail_open(deployments, deployments, frozenset(), frozenset(), frozenset()) is False
def test_chain_allows_fail_open_true_when_no_required_tag_is_known_at_all():
@ -1384,7 +1390,10 @@ def test_chain_allows_fail_open_true_when_no_required_tag_is_known_at_all():
deployments = [
{"model_info": {"allow_fail_open": True}, "litellm_params": {"tags": ["default", "reasoning_type:low"]}},
]
assert _chain_allows_fail_open(deployments, frozenset(), frozenset({"reasoning_type:high"}), frozenset()) is True
assert (
_chain_allows_fail_open(deployments, deployments, frozenset(), frozenset({"reasoning_type:high"}), frozenset())
is True
)
def test_unknown_required_tag_hides_an_answer_denies_fail_open():
@ -1405,7 +1414,11 @@ def test_unknown_required_tag_hides_an_answer_denies_fail_open():
# fail-open must be denied even though the flag is set on the group.
assert (
_chain_allows_fail_open(
deployments, frozenset(), frozenset({"region:us-east", "totally-invented-tag-nobody-has"}), frozenset()
deployments,
deployments,
frozenset(),
frozenset({"region:us-east", "totally-invented-tag-nobody-has"}),
frozenset(),
)
is False
)
@ -1432,7 +1445,9 @@ def test_unknown_required_tag_allows_fail_open_when_no_answer_is_hidden():
# carries both, so this is a genuinely unsatisfiable combination, not an invented
# tag masking a narrower answer. Fail-open must proceed normally.
assert (
_chain_allows_fail_open(deployments, frozenset(), frozenset({"region:us-east", "region:eu"}), frozenset())
_chain_allows_fail_open(
deployments, deployments, frozenset(), frozenset({"region:us-east", "region:eu"}), frozenset()
)
is True
)
@ -1494,8 +1509,8 @@ def test_chain_allows_fail_open_true_when_prefixed_unknown_required_tag_is_confi
},
]
required_set = frozenset({"provider:anthropic", "typo-tag"})
assert _chain_allows_fail_open(deployments, frozenset(), required_set, frozenset()) is False
assert _chain_allows_fail_open(deployments, frozenset(), required_set, required_set) is True
assert _chain_allows_fail_open(deployments, deployments, frozenset(), required_set, frozenset()) is False
assert _chain_allows_fail_open(deployments, deployments, frozenset(), required_set, required_set) is True
# --- get_deployments_for_tag required-AND ("&") integration tests ---
@ -1763,6 +1778,141 @@ async def test_required_and_combined_with_positive_unmatched_raises_by_default()
# --- get_deployments_for_tag allow_fail_open integration tests ---
def _fail_open_router(opted_in_group: str | None = "chat", route: str = "model") -> litellm.Router:
model_name: Final = "chat-*" if route == "wildcard" else "chat"
team_info: Final = {"team_id": "team-a", "team_public_model_name": "public-chat"} if route == "team" else {}
return litellm.Router(
model_list=[
{
"model_name": model_name if opted_in_group in (None, "chat") else opted_in_group,
"litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused", "tags": ["vision"]},
"model_info": {"id": "opted-in", "allow_fail_open": opted_in_group is not None, **team_info},
},
{
"model_name": model_name,
"litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused", "tags": ["region:eu"]},
"model_info": {"id": "remaining", **team_info},
},
],
routing_groups=(
[{"group_name": "public-chat", "models": ["chat"], "routing_strategy": "simple-shuffle"}]
if route == "routing_group"
else None
),
model_group_alias={"public-chat": "chat"} if route == "alias" else {},
enable_tag_filtering=True,
optional_pre_call_checks=["responses_api_deployment_check"],
)
def _response_id_for_remaining_deployment() -> str:
return ResponsesAPIRequestUtils._build_responses_api_response_id(
custom_llm_provider="openai", model_id="remaining", response_id="resp_previous"
)
@pytest.mark.asyncio()
@pytest.mark.parametrize("narrowing", ("affinity", "cooldown"))
@pytest.mark.parametrize("tags", (("component:chat-search",), ("&vision",), ("!region:eu",)))
@pytest.mark.parametrize("route", ("model", "routing_group", "wildcard", "team", "alias"))
async def test_tag_fail_open_survives_deployment_narrowing(
narrowing: Literal["affinity", "cooldown"], tags: tuple[str, ...], route: str
) -> None:
router: Final = _fail_open_router(route=route)
requested_model: Final = "chat" if route == "model" else "chat-test" if route == "wildcard" else "public-chat"
with patch("time.time", return_value=1_700_000_000.0):
if narrowing == "cooldown":
router.cooldown_cache.add_deployment_to_cooldown(
model_id="opted-in",
original_exception=RuntimeError("unavailable"),
exception_status=500,
cooldown_time=60,
)
candidates: Final = await router.async_get_healthy_deployments(
model=requested_model,
request_kwargs={
"litellm_metadata": {"tags": list(tags), "inherited_tags": [], "user_api_key_team_id": "team-a"},
**(
{"previous_response_id": _response_id_for_remaining_deployment()} if narrowing == "affinity" else {}
),
},
)
assert isinstance(candidates, (list, tuple))
assert tuple(deployment["model_info"]["id"] for deployment in candidates) == ("remaining",)
@pytest.mark.asyncio()
@pytest.mark.parametrize("route", ("routing_group", "wildcard", "team", "alias"))
async def test_tag_fail_open_resolves_public_model_names(route: str) -> None:
router: Final = _fail_open_router(route=route)
candidates: Final = await router.async_get_healthy_deployments(
model="chat-test" if route == "wildcard" else "public-chat",
request_kwargs={
"litellm_metadata": {"tags": ["component:chat-search"], "user_api_key_team_id": "team-a"},
},
)
assert isinstance(candidates, (list, tuple))
assert {deployment["model_info"]["id"] for deployment in candidates} == {"opted-in", "remaining"}
@pytest.mark.asyncio()
@pytest.mark.parametrize("opted_in_group", (None, "other-group"))
async def test_tag_fail_open_requires_opt_in_in_the_requested_group(opted_in_group: str | None) -> None:
router: Final = _fail_open_router(opted_in_group)
with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value):
await router.async_get_healthy_deployments(
model="chat",
request_kwargs={
"previous_response_id": _response_id_for_remaining_deployment(),
"litellm_metadata": {"tags": ["component:chat-search"], "inherited_tags": []},
},
)
@pytest.mark.asyncio()
async def test_tag_fail_open_does_not_use_another_teams_opt_in() -> None:
router: Final = litellm.Router(
model_list=[
{
"model_name": "chat",
"litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "unused"},
"model_info": {
"id": team_id,
"team_id": team_id,
"team_public_model_name": "public-chat",
"allow_fail_open": team_id == "team-a",
},
}
for team_id in ("team-a", "team-b")
],
enable_tag_filtering=True,
)
with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value):
await router.async_get_healthy_deployments(
model="public-chat",
request_kwargs={
"litellm_metadata": {"tags": ["component:chat-search"], "user_api_key_team_id": "team-b"},
},
)
@pytest.mark.asyncio()
@pytest.mark.parametrize("inherited_tag", ("&vision", "!region:eu"))
async def test_tag_fail_open_preserves_inherited_policy_after_affinity(inherited_tag: str) -> None:
router: Final = _fail_open_router()
with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_tag_routing.value):
await router.async_get_healthy_deployments(
model="chat",
request_kwargs={
"previous_response_id": _response_id_for_remaining_deployment(),
"litellm_metadata": {
"tags": [inherited_tag, "component:chat-search"],
"inherited_tags": [inherited_tag],
},
},
)
@pytest.mark.asyncio()
async def test_allow_fail_open_required_and_unmatched_falls_back_to_default_pool():
router = litellm.Router(