scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-17 04:34:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 171

fix(router): make tag-based routing retryable when matching deployment is cooled down

Browse source 
When enable_tag_filtering is on and a deployment matching the request's tags exists for the model but is not in the currently-healthy set (e.g. in cooldown after upstream errors) - while other same-model_name deployments for other tags are still healthy - get_deployments_for_tag raised no_deployments_with_tag_routing. The proxy maps that to a 401 ("Not allowed to access model due to tags configuration"), which clients treat as an auth failure instead of a transient, retryable condition.

Now, when matching deployment(s) exist for the model but none are currently healthy, return an empty list so the router raises its standard RouterRateLimitError ("No deployments available, try again in N seconds"), which the proxy maps to a retryable 429. The tags error is still raised only when no deployment matches the request at all (genuine misconfig / no access).

Fixes #15016

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
poi1649 2026-06-17 13:58:25 +02:00
parent 13b590c8ec
commit 4c89930146
3 changed files with 216 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/router.py
View file

@ -10202,6 +10202,11 @@ class Router:
request_kwargs=request_kwargs,
healthy_deployments=healthy_deployments,
metadata_variable_name=self._get_metadata_variable_name_from_kwargs(request_kwargs),
# Authorized candidate set before cooldown filtering. Used so a tag
# match that exists but is only cooled down is treated as retryable,
# without inspecting deployments the request isn't authorized to route
# to. https://github.com/BerriAI/litellm/issues/15016
routing_candidates=_pre_cooldown_deployments,
)
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)

79
litellm/router_strategy/tag_based_routing.py
View file

@ -149,12 +149,82 @@ def _ban_only_base_pool(
return defaults if defaults else list(deployments)
def _model_has_matching_deployment(
candidate_deployments: list,
request_tags: Optional[list[str]],
header_strings: list[str],
match_any: bool,
) -> bool:
"""
Whether any deployment in *candidate_deployments* would be selected for this
request - via exact tag, tag_regex, or a "default" tag.
The caller passes the routing candidate set (already narrowed by team / access
/ health filters), so this check never looks beyond what the request is
authorized to route to. Used to distinguish "matching deployment(s) exist but
are temporarily unavailable, e.g. all in cooldown" (retryable) from "no
deployment is configured for this request" (a genuine 401).
"""
for deployment in candidate_deployments or []:
if (
_match_deployment(
deployment=deployment,
request_tags=request_tags,
header_strings=header_strings,
match_any=match_any,
)
is not None
):
return True
deployment_tags = deployment.get("litellm_params", {}).get("tags")
if deployment_tags and "default" in deployment_tags:
return True
return False
def _handle_no_tag_match(
candidate_deployments: list,
model: str,
request_tags: Optional[list[str]],
header_strings: list[str],
match_any: bool,
) -> list:
"""
Called when tag filtering matched no healthy deployment.
If a deployment matching this request exists among the (already authorized)
routing candidates but is temporarily unavailable (e.g. all matches in
cooldown), return [] so the router raises its standard, retryable
RouterRateLimitError (mapped to 429) instead of this tags error (mapped to
401). Only raise when no candidate matches this request at all (genuine
misconfiguration / no access).
https://github.com/BerriAI/litellm/issues/15016
"""
if _model_has_matching_deployment(
candidate_deployments=candidate_deployments,
request_tags=request_tags,
header_strings=header_strings,
match_any=match_any,
):
verbose_logger.debug(
"get_deployments_for_tag: matching deployment(s) exist for model=%s "
"but none are currently healthy; returning [] so the router raises a "
"retryable error",
model,
)
return []
raise ValueError(
f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model={model} and tags={request_tags}"
)
async def get_deployments_for_tag(
llm_router_instance: LitellmRouter,
model: str, # used to raise the correct error
healthy_deployments: Union[list[Any], dict[Any, Any]],
request_kwargs: Optional[dict[Any, Any]] = None,
metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata",
routing_candidates: Optional[list] = None,
):
"""
Returns a list of deployments that match the requested model and tags in the request.
@ -239,9 +309,12 @@ async def get_deployments_for_tag(
default_deployments.append(deployment)
if len(new_healthy_deployments) == 0 and len(default_deployments) == 0:
raise ValueError(
f"{RouterErrors.no_deployments_with_tag_routing.value}."
f" Passed model={model} and tags={request_tags}"
return _handle_no_tag_match(
candidate_deployments=routing_candidates or [],
model=model,
request_tags=positive_tags,
header_strings=header_strings,
match_any=match_any,
)
return new_healthy_deployments if len(new_healthy_deployments) > 0 else default_deployments

135
tests/test_litellm/router_strategy/test_router_tag_routing.py
View file

@ -1019,3 +1019,138 @@ async def test_negation_removes_tag_regex_deployment_falls_to_ban_only():
mock_response="hi",
)
assert response._hidden_params["model_id"] == "openai-deployment"
@pytest.mark.asyncio()
async def test_tag_routing_returns_empty_when_matching_deployment_unavailable():
"""
Regression for https://github.com/BerriAI/litellm/issues/15016.
When a deployment matching the request tags exists for the model but is not
in the currently-healthy set (e.g. in cooldown after upstream errors) while
other same-model_name deployments (for other tags) are still healthy, tag
routing must return [] so the router raises its standard retryable
RouterRateLimitError (mapped to 429) instead of the misleading
no_deployments_with_tag_routing error (mapped to 401).
"""
from litellm.router_strategy.tag_based_routing import get_deployments_for_tag
router = litellm.Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {"model": "gpt-4o", "tags": ["teamA"]},
"model_info": {"id": "team-a-model"},
},
{
"model_name": "gpt-4",
"litellm_params": {"model": "gpt-4o-mini", "tags": ["teamB"]},
"model_info": {"id": "team-b-model"},
},
],
enable_tag_filtering=True,
)
# Simulate the teamA deployment being in cooldown: only teamB remains in the
# healthy candidate set, but the request targets teamA.
healthy_deployments = [
d
for d in (router.get_model_list(model_name="gpt-4") or [])
if "teamB" in (d.get("litellm_params", {}).get("tags") or [])
]
# routing_candidates is the authorized, pre-cooldown candidate set (teamA +
# teamB); teamA is the cooled match that is absent from healthy_deployments.
result = await get_deployments_for_tag(
llm_router_instance=router,
model="gpt-4",
healthy_deployments=healthy_deployments,
request_kwargs={"metadata": {"tags": ["teamA"]}},
routing_candidates=router.get_model_list(model_name="gpt-4"),
)
assert result == []
@pytest.mark.asyncio()
async def test_tag_routing_still_raises_when_no_matching_deployment_exists():
"""
When no deployment carries the requested tag at all (genuine misconfiguration
/ no access), tag routing still raises no_deployments_with_tag_routing rather
than returning an empty (retryable) result.
"""
from litellm.router_strategy.tag_based_routing import get_deployments_for_tag
from litellm.types.router import RouterErrors
router = litellm.Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {"model": "gpt-4o", "tags": ["teamA"]},
"model_info": {"id": "team-a-model"},
},
],
enable_tag_filtering=True,
)
healthy_deployments = router.get_model_list(model_name="gpt-4")
with pytest.raises(ValueError) as exc_info:
await get_deployments_for_tag(
llm_router_instance=router,
model="gpt-4",
healthy_deployments=healthy_deployments,
request_kwargs={"metadata": {"tags": ["nonexistent-tag"]}},
routing_candidates=router.get_model_list(model_name="gpt-4"),
)
assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value)
@pytest.mark.asyncio()
async def test_tag_routing_does_not_leak_unauthorized_tags():
"""
Security: the retryable-vs-error decision must only consider deployments the
request is authorized to route to (routing_candidates). A tag that exists only
on a deployment outside that set (e.g. another team's) must still raise
no_deployments_with_tag_routing - identical to a non-existent tag - so an
authenticated caller cannot probe guessed tags and tell them apart.
https://github.com/BerriAI/litellm/issues/15016
"""
from litellm.router_strategy.tag_based_routing import get_deployments_for_tag
from litellm.types.router import RouterErrors
router = litellm.Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {"model": "gpt-4o", "tags": ["teamA"]},
"model_info": {"id": "team-a-model"},
},
{
"model_name": "gpt-4",
"litellm_params": {"model": "gpt-4o-mini", "tags": ["teamB"]},
"model_info": {"id": "team-b-model"},
},
],
enable_tag_filtering=True,
)
teamA_only = [
d
for d in (router.get_model_list(model_name="gpt-4") or [])
if "teamA" in (d.get("litellm_params", {}).get("tags") or [])
]
# The request probes "teamB" (which exists in the config) but is only
# authorized for teamA -> must raise, not return a retryable [].
with pytest.raises(ValueError) as exc_info:
await get_deployments_for_tag(
llm_router_instance=router,
model="gpt-4",
healthy_deployments=teamA_only,
request_kwargs={"metadata": {"tags": ["teamB"]}},
routing_candidates=teamA_only,
)
assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value)