From 4c899301468ee2632033919027fd46f0f1914c0e Mon Sep 17 00:00:00 2001 From: poi1649 Date: Wed, 17 Jun 2026 13:58:25 +0200 Subject: [PATCH] fix(router): make tag-based routing retryable when matching deployment is cooled down When enable_tag_filtering is on and a deployment matching the request's tags exists for the model but is not in the currently-healthy set (e.g. in cooldown after upstream errors) - while other same-model_name deployments for other tags are still healthy - get_deployments_for_tag raised no_deployments_with_tag_routing. The proxy maps that to a 401 ("Not allowed to access model due to tags configuration"), which clients treat as an auth failure instead of a transient, retryable condition. Now, when matching deployment(s) exist for the model but none are currently healthy, return an empty list so the router raises its standard RouterRateLimitError ("No deployments available, try again in N seconds"), which the proxy maps to a retryable 429. The tags error is still raised only when no deployment matches the request at all (genuine misconfig / no access). Fixes #15016 Co-Authored-By: Claude Opus 4.8 --- litellm/router.py | 5 + litellm/router_strategy/tag_based_routing.py | 79 +++++++++- .../test_router_tag_routing.py | 135 ++++++++++++++++++ 3 files changed, 216 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 2d66bc3158d..738bdb3d8a6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10202,6 +10202,11 @@ class Router: request_kwargs=request_kwargs, healthy_deployments=healthy_deployments, metadata_variable_name=self._get_metadata_variable_name_from_kwargs(request_kwargs), + # Authorized candidate set before cooldown filtering. Used so a tag + # match that exists but is only cooled down is treated as retryable, + # without inspecting deployments the request isn't authorized to route + # to. https://github.com/BerriAI/litellm/issues/15016 + routing_candidates=_pre_cooldown_deployments, ) ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index 6ca4e1de322..609eb143a12 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -149,12 +149,82 @@ def _ban_only_base_pool( return defaults if defaults else list(deployments) +def _model_has_matching_deployment( + candidate_deployments: list, + request_tags: Optional[list[str]], + header_strings: list[str], + match_any: bool, +) -> bool: + """ + Whether any deployment in *candidate_deployments* would be selected for this + request - via exact tag, tag_regex, or a "default" tag. + + The caller passes the routing candidate set (already narrowed by team / access + / health filters), so this check never looks beyond what the request is + authorized to route to. Used to distinguish "matching deployment(s) exist but + are temporarily unavailable, e.g. all in cooldown" (retryable) from "no + deployment is configured for this request" (a genuine 401). + """ + for deployment in candidate_deployments or []: + if ( + _match_deployment( + deployment=deployment, + request_tags=request_tags, + header_strings=header_strings, + match_any=match_any, + ) + is not None + ): + return True + deployment_tags = deployment.get("litellm_params", {}).get("tags") + if deployment_tags and "default" in deployment_tags: + return True + return False + + +def _handle_no_tag_match( + candidate_deployments: list, + model: str, + request_tags: Optional[list[str]], + header_strings: list[str], + match_any: bool, +) -> list: + """ + Called when tag filtering matched no healthy deployment. + + If a deployment matching this request exists among the (already authorized) + routing candidates but is temporarily unavailable (e.g. all matches in + cooldown), return [] so the router raises its standard, retryable + RouterRateLimitError (mapped to 429) instead of this tags error (mapped to + 401). Only raise when no candidate matches this request at all (genuine + misconfiguration / no access). + https://github.com/BerriAI/litellm/issues/15016 + """ + if _model_has_matching_deployment( + candidate_deployments=candidate_deployments, + request_tags=request_tags, + header_strings=header_strings, + match_any=match_any, + ): + verbose_logger.debug( + "get_deployments_for_tag: matching deployment(s) exist for model=%s " + "but none are currently healthy; returning [] so the router raises a " + "retryable error", + model, + ) + return [] + raise ValueError( + f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model={model} and tags={request_tags}" + ) + + async def get_deployments_for_tag( llm_router_instance: LitellmRouter, model: str, # used to raise the correct error healthy_deployments: Union[list[Any], dict[Any, Any]], request_kwargs: Optional[dict[Any, Any]] = None, metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata", + routing_candidates: Optional[list] = None, ): """ Returns a list of deployments that match the requested model and tags in the request. @@ -239,9 +309,12 @@ async def get_deployments_for_tag( default_deployments.append(deployment) if len(new_healthy_deployments) == 0 and len(default_deployments) == 0: - raise ValueError( - f"{RouterErrors.no_deployments_with_tag_routing.value}." - f" Passed model={model} and tags={request_tags}" + return _handle_no_tag_match( + candidate_deployments=routing_candidates or [], + model=model, + request_tags=positive_tags, + header_strings=header_strings, + match_any=match_any, ) return new_healthy_deployments if len(new_healthy_deployments) > 0 else default_deployments diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index eb289095c51..9e8afbf47d4 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -1019,3 +1019,138 @@ async def test_negation_removes_tag_regex_deployment_falls_to_ban_only(): mock_response="hi", ) assert response._hidden_params["model_id"] == "openai-deployment" + + +@pytest.mark.asyncio() +async def test_tag_routing_returns_empty_when_matching_deployment_unavailable(): + """ + Regression for https://github.com/BerriAI/litellm/issues/15016. + + When a deployment matching the request tags exists for the model but is not + in the currently-healthy set (e.g. in cooldown after upstream errors) while + other same-model_name deployments (for other tags) are still healthy, tag + routing must return [] so the router raises its standard retryable + RouterRateLimitError (mapped to 429) instead of the misleading + no_deployments_with_tag_routing error (mapped to 401). + """ + from litellm.router_strategy.tag_based_routing import get_deployments_for_tag + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o", "tags": ["teamA"]}, + "model_info": {"id": "team-a-model"}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o-mini", "tags": ["teamB"]}, + "model_info": {"id": "team-b-model"}, + }, + ], + enable_tag_filtering=True, + ) + + # Simulate the teamA deployment being in cooldown: only teamB remains in the + # healthy candidate set, but the request targets teamA. + healthy_deployments = [ + d + for d in (router.get_model_list(model_name="gpt-4") or []) + if "teamB" in (d.get("litellm_params", {}).get("tags") or []) + ] + + # routing_candidates is the authorized, pre-cooldown candidate set (teamA + + # teamB); teamA is the cooled match that is absent from healthy_deployments. + result = await get_deployments_for_tag( + llm_router_instance=router, + model="gpt-4", + healthy_deployments=healthy_deployments, + request_kwargs={"metadata": {"tags": ["teamA"]}}, + routing_candidates=router.get_model_list(model_name="gpt-4"), + ) + + assert result == [] + + +@pytest.mark.asyncio() +async def test_tag_routing_still_raises_when_no_matching_deployment_exists(): + """ + When no deployment carries the requested tag at all (genuine misconfiguration + / no access), tag routing still raises no_deployments_with_tag_routing rather + than returning an empty (retryable) result. + """ + from litellm.router_strategy.tag_based_routing import get_deployments_for_tag + from litellm.types.router import RouterErrors + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o", "tags": ["teamA"]}, + "model_info": {"id": "team-a-model"}, + }, + ], + enable_tag_filtering=True, + ) + + healthy_deployments = router.get_model_list(model_name="gpt-4") + + with pytest.raises(ValueError) as exc_info: + await get_deployments_for_tag( + llm_router_instance=router, + model="gpt-4", + healthy_deployments=healthy_deployments, + request_kwargs={"metadata": {"tags": ["nonexistent-tag"]}}, + routing_candidates=router.get_model_list(model_name="gpt-4"), + ) + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value) + + +@pytest.mark.asyncio() +async def test_tag_routing_does_not_leak_unauthorized_tags(): + """ + Security: the retryable-vs-error decision must only consider deployments the + request is authorized to route to (routing_candidates). A tag that exists only + on a deployment outside that set (e.g. another team's) must still raise + no_deployments_with_tag_routing - identical to a non-existent tag - so an + authenticated caller cannot probe guessed tags and tell them apart. + https://github.com/BerriAI/litellm/issues/15016 + """ + from litellm.router_strategy.tag_based_routing import get_deployments_for_tag + from litellm.types.router import RouterErrors + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o", "tags": ["teamA"]}, + "model_info": {"id": "team-a-model"}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o-mini", "tags": ["teamB"]}, + "model_info": {"id": "team-b-model"}, + }, + ], + enable_tag_filtering=True, + ) + + teamA_only = [ + d + for d in (router.get_model_list(model_name="gpt-4") or []) + if "teamA" in (d.get("litellm_params", {}).get("tags") or []) + ] + + # The request probes "teamB" (which exists in the config) but is only + # authorized for teamA -> must raise, not return a retryable []. + with pytest.raises(ValueError) as exc_info: + await get_deployments_for_tag( + llm_router_instance=router, + model="gpt-4", + healthy_deployments=teamA_only, + request_kwargs={"metadata": {"tags": ["teamB"]}}, + routing_candidates=teamA_only, + ) + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value)