diff --git a/litellm/router.py b/litellm/router.py index 2d66bc3158d..738bdb3d8a6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -10202,6 +10202,11 @@ class Router: request_kwargs=request_kwargs, healthy_deployments=healthy_deployments, metadata_variable_name=self._get_metadata_variable_name_from_kwargs(request_kwargs), + # Authorized candidate set before cooldown filtering. Used so a tag + # match that exists but is only cooled down is treated as retryable, + # without inspecting deployments the request isn't authorized to route + # to. https://github.com/BerriAI/litellm/issues/15016 + routing_candidates=_pre_cooldown_deployments, ) ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2) diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py index 6ca4e1de322..609eb143a12 100644 --- a/litellm/router_strategy/tag_based_routing.py +++ b/litellm/router_strategy/tag_based_routing.py @@ -149,12 +149,82 @@ def _ban_only_base_pool( return defaults if defaults else list(deployments) +def _model_has_matching_deployment( + candidate_deployments: list, + request_tags: Optional[list[str]], + header_strings: list[str], + match_any: bool, +) -> bool: + """ + Whether any deployment in *candidate_deployments* would be selected for this + request - via exact tag, tag_regex, or a "default" tag. + + The caller passes the routing candidate set (already narrowed by team / access + / health filters), so this check never looks beyond what the request is + authorized to route to. Used to distinguish "matching deployment(s) exist but + are temporarily unavailable, e.g. all in cooldown" (retryable) from "no + deployment is configured for this request" (a genuine 401). + """ + for deployment in candidate_deployments or []: + if ( + _match_deployment( + deployment=deployment, + request_tags=request_tags, + header_strings=header_strings, + match_any=match_any, + ) + is not None + ): + return True + deployment_tags = deployment.get("litellm_params", {}).get("tags") + if deployment_tags and "default" in deployment_tags: + return True + return False + + +def _handle_no_tag_match( + candidate_deployments: list, + model: str, + request_tags: Optional[list[str]], + header_strings: list[str], + match_any: bool, +) -> list: + """ + Called when tag filtering matched no healthy deployment. + + If a deployment matching this request exists among the (already authorized) + routing candidates but is temporarily unavailable (e.g. all matches in + cooldown), return [] so the router raises its standard, retryable + RouterRateLimitError (mapped to 429) instead of this tags error (mapped to + 401). Only raise when no candidate matches this request at all (genuine + misconfiguration / no access). + https://github.com/BerriAI/litellm/issues/15016 + """ + if _model_has_matching_deployment( + candidate_deployments=candidate_deployments, + request_tags=request_tags, + header_strings=header_strings, + match_any=match_any, + ): + verbose_logger.debug( + "get_deployments_for_tag: matching deployment(s) exist for model=%s " + "but none are currently healthy; returning [] so the router raises a " + "retryable error", + model, + ) + return [] + raise ValueError( + f"{RouterErrors.no_deployments_with_tag_routing.value}. Passed model={model} and tags={request_tags}" + ) + + async def get_deployments_for_tag( llm_router_instance: LitellmRouter, model: str, # used to raise the correct error healthy_deployments: Union[list[Any], dict[Any, Any]], request_kwargs: Optional[dict[Any, Any]] = None, metadata_variable_name: Literal["metadata", "litellm_metadata"] = "metadata", + routing_candidates: Optional[list] = None, ): """ Returns a list of deployments that match the requested model and tags in the request. @@ -239,9 +309,12 @@ async def get_deployments_for_tag( default_deployments.append(deployment) if len(new_healthy_deployments) == 0 and len(default_deployments) == 0: - raise ValueError( - f"{RouterErrors.no_deployments_with_tag_routing.value}." - f" Passed model={model} and tags={request_tags}" + return _handle_no_tag_match( + candidate_deployments=routing_candidates or [], + model=model, + request_tags=positive_tags, + header_strings=header_strings, + match_any=match_any, ) return new_healthy_deployments if len(new_healthy_deployments) > 0 else default_deployments diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py index eb289095c51..9e8afbf47d4 100644 --- a/tests/test_litellm/router_strategy/test_router_tag_routing.py +++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py @@ -1019,3 +1019,138 @@ async def test_negation_removes_tag_regex_deployment_falls_to_ban_only(): mock_response="hi", ) assert response._hidden_params["model_id"] == "openai-deployment" + + +@pytest.mark.asyncio() +async def test_tag_routing_returns_empty_when_matching_deployment_unavailable(): + """ + Regression for https://github.com/BerriAI/litellm/issues/15016. + + When a deployment matching the request tags exists for the model but is not + in the currently-healthy set (e.g. in cooldown after upstream errors) while + other same-model_name deployments (for other tags) are still healthy, tag + routing must return [] so the router raises its standard retryable + RouterRateLimitError (mapped to 429) instead of the misleading + no_deployments_with_tag_routing error (mapped to 401). + """ + from litellm.router_strategy.tag_based_routing import get_deployments_for_tag + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o", "tags": ["teamA"]}, + "model_info": {"id": "team-a-model"}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o-mini", "tags": ["teamB"]}, + "model_info": {"id": "team-b-model"}, + }, + ], + enable_tag_filtering=True, + ) + + # Simulate the teamA deployment being in cooldown: only teamB remains in the + # healthy candidate set, but the request targets teamA. + healthy_deployments = [ + d + for d in (router.get_model_list(model_name="gpt-4") or []) + if "teamB" in (d.get("litellm_params", {}).get("tags") or []) + ] + + # routing_candidates is the authorized, pre-cooldown candidate set (teamA + + # teamB); teamA is the cooled match that is absent from healthy_deployments. + result = await get_deployments_for_tag( + llm_router_instance=router, + model="gpt-4", + healthy_deployments=healthy_deployments, + request_kwargs={"metadata": {"tags": ["teamA"]}}, + routing_candidates=router.get_model_list(model_name="gpt-4"), + ) + + assert result == [] + + +@pytest.mark.asyncio() +async def test_tag_routing_still_raises_when_no_matching_deployment_exists(): + """ + When no deployment carries the requested tag at all (genuine misconfiguration + / no access), tag routing still raises no_deployments_with_tag_routing rather + than returning an empty (retryable) result. + """ + from litellm.router_strategy.tag_based_routing import get_deployments_for_tag + from litellm.types.router import RouterErrors + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o", "tags": ["teamA"]}, + "model_info": {"id": "team-a-model"}, + }, + ], + enable_tag_filtering=True, + ) + + healthy_deployments = router.get_model_list(model_name="gpt-4") + + with pytest.raises(ValueError) as exc_info: + await get_deployments_for_tag( + llm_router_instance=router, + model="gpt-4", + healthy_deployments=healthy_deployments, + request_kwargs={"metadata": {"tags": ["nonexistent-tag"]}}, + routing_candidates=router.get_model_list(model_name="gpt-4"), + ) + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value) + + +@pytest.mark.asyncio() +async def test_tag_routing_does_not_leak_unauthorized_tags(): + """ + Security: the retryable-vs-error decision must only consider deployments the + request is authorized to route to (routing_candidates). A tag that exists only + on a deployment outside that set (e.g. another team's) must still raise + no_deployments_with_tag_routing - identical to a non-existent tag - so an + authenticated caller cannot probe guessed tags and tell them apart. + https://github.com/BerriAI/litellm/issues/15016 + """ + from litellm.router_strategy.tag_based_routing import get_deployments_for_tag + from litellm.types.router import RouterErrors + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o", "tags": ["teamA"]}, + "model_info": {"id": "team-a-model"}, + }, + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4o-mini", "tags": ["teamB"]}, + "model_info": {"id": "team-b-model"}, + }, + ], + enable_tag_filtering=True, + ) + + teamA_only = [ + d + for d in (router.get_model_list(model_name="gpt-4") or []) + if "teamA" in (d.get("litellm_params", {}).get("tags") or []) + ] + + # The request probes "teamB" (which exists in the config) but is only + # authorized for teamA -> must raise, not return a retryable []. + with pytest.raises(ValueError) as exc_info: + await get_deployments_for_tag( + llm_router_instance=router, + model="gpt-4", + healthy_deployments=teamA_only, + request_kwargs={"metadata": {"tags": ["teamB"]}}, + routing_candidates=teamA_only, + ) + + assert RouterErrors.no_deployments_with_tag_routing.value in str(exc_info.value)