fix(health): show deployments reached through wildcard and alias allowlists, and by id when an alias shadows the name

This commit is contained in:
mateo-berri 2026-10-28 19:18:42 -07:00
parent e9e76077d9
commit 8d9eab02fc
3 changed files with 233 additions and 27 deletions

View file

@ -7,7 +7,9 @@ import secrets
import time
import traceback
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from itertools import chain, groupby
from types import MappingProxyType
from typing import Final, Literal, TypedDict
@ -81,6 +83,7 @@ from litellm.router_utils.clientside_credential_handler import (
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path
clientside_credential_keys,
)
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.secret_managers.main import get_secret_bool
from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields
from litellm.types.router import reject_server_owned_wif_fields
@ -255,7 +258,7 @@ def get_callback_identifier(callback):
router: Final = APIRouter()
_OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
_DEPLOYMENT_LIST: Final = TypeAdapter(tuple[object, ...])
_OBJECTS: Final = TypeAdapter(tuple[object, ...])
services = (
Literal[
"slack_budget_alerts",
@ -1101,34 +1104,139 @@ async def _caller_aliases(user_api_key_dict: UserAPIKeyAuth) -> CallerAliases:
)
def _served_model_group(model_name: str, aliases: CallerAliases, llm_router: Router | None) -> str:
def _model_auth_checks(model_name: str, aliases: CallerAliases) -> str:
return resolve_model_group_alias(aliases.router, model_name) or model_name
def _deployment_id(deployment: object) -> str | None:
match deployment:
case {"model_info": {"id": str(model_id)}}:
return model_id
case _:
return None
def _granted_names(user_api_key_dict: UserAPIKeyAuth) -> frozenset[str]:
grants: Final = (
*_OBJECTS.validate_python(user_api_key_dict.models),
*_OBJECTS.validate_python(user_api_key_dict.team_models),
)
return frozenset(name for name in grants if isinstance(name, str))
def _served_group(model_name: str, aliases: CallerAliases, llm_router: Router | None) -> str:
dispatched: Final = dispatched_model(model_name, aliases)
return llm_router.routable_model_group(dispatched) if llm_router is not None else dispatched
async def _model_groups_key_may_reach(
llm_model_list: Sequence[object], user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None
) -> frozenset[str]:
requested_names: Final = frozenset[str]().union(
*(_names_caller_may_request(deployment, user_api_key_dict.team_id) for deployment in llm_model_list)
def _name_id_pairs(deployment: object, team_id: str | None) -> tuple[tuple[str, str], ...]:
model_id: Final = _deployment_id(deployment)
if model_id is None:
return ()
return tuple((name, model_id) for name in _names_caller_may_request(deployment, team_id) | {model_id})
def _ids_by_requested_name(llm_model_list: Sequence[object], team_id: str | None) -> Mapping[str, frozenset[str]]:
pairs: Final = sorted(chain.from_iterable(_name_id_pairs(deployment, team_id) for deployment in llm_model_list))
return MappingProxyType(
{name: frozenset(model_id for _, model_id in group) for name, group in groupby(pairs, key=lambda pair: pair[0])}
)
aliases: Final = await _caller_aliases(user_api_key_dict)
def _ids_serving_group(
group: str,
ids_by_name: Mapping[str, frozenset[str]],
user_api_key_dict: UserAPIKeyAuth,
llm_router: Router | None,
) -> frozenset[str]:
named: Final = ids_by_name.get(group)
if named is not None or llm_router is None:
return named or frozenset()
routed: Final = llm_router.get_routable_deployments(
group, {"metadata": {"user_api_key_team_id": user_api_key_dict.team_id}}
)
return frozenset(model_id for model_id in map(_deployment_id, routed) if model_id is not None)
async def _names_key_may_request(
names: Iterable[str], aliases: CallerAliases, user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None
) -> frozenset[str]:
return frozenset(
[
_served_model_group(name, aliases, llm_router)
for name in requested_names
if await _key_may_call_model(name, user_api_key_dict, llm_router)
name
for name in names
if await _key_may_call_model(_model_auth_checks(name, aliases), user_api_key_dict, llm_router)
]
)
def _ids_serving(
groups: Iterable[str],
ids_by_name: Mapping[str, frozenset[str]],
user_api_key_dict: UserAPIKeyAuth,
llm_router: Router | None,
) -> frozenset[str]:
return frozenset[str]().union(
*(_ids_serving_group(group, ids_by_name, user_api_key_dict, llm_router) for group in groups)
)
def _shadowed_ids(
llm_model_list: Sequence[object], admitted_names: frozenset[str], reached_ids: frozenset[str], team_id: str | None
) -> frozenset[str]:
ids_of_admitted_names: Final = (
_deployment_id(deployment)
for deployment in llm_model_list
if not admitted_names.isdisjoint(_names_caller_may_request(deployment, team_id))
)
return frozenset(model_id for model_id in ids_of_admitted_names if model_id is not None) - reached_ids
@dataclass(frozen=True, slots=True)
class _ReachableDeployments:
model_ids: frozenset[str]
groups: frozenset[str]
def include(self, deployment: object, team_id: str | None) -> bool:
model_id: Final = _deployment_id(deployment)
if model_id is None:
return not self.groups.isdisjoint(_names_caller_may_request(deployment, team_id))
return model_id in self.model_ids
_NOTHING_REACHABLE: Final = _ReachableDeployments(frozenset(), frozenset())
async def _deployments_key_may_reach(
llm_model_list: Sequence[object], user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None
) -> _ReachableDeployments:
aliases: Final = await _caller_aliases(user_api_key_dict)
candidate_names: Final = _granted_names(user_api_key_dict).union(
*(_names_caller_may_request(deployment, user_api_key_dict.team_id) for deployment in llm_model_list)
)
admitted_names: Final = await _names_key_may_request(candidate_names, aliases, user_api_key_dict, llm_router)
groups: Final = frozenset(_served_group(name, aliases, llm_router) for name in admitted_names)
ids_by_name: Final = _ids_by_requested_name(llm_model_list, user_api_key_dict.team_id)
reached_ids: Final = _ids_serving(groups, ids_by_name, user_api_key_dict, llm_router)
admitted_ids: Final = await _names_key_may_request(
_shadowed_ids(llm_model_list, admitted_names, reached_ids, user_api_key_dict.team_id),
aliases,
user_api_key_dict,
llm_router,
)
id_groups: Final = frozenset(_served_group(model_id, aliases, llm_router) for model_id in admitted_ids)
return _ReachableDeployments(
reached_ids | _ids_serving(id_groups, ids_by_name, user_api_key_dict, llm_router), groups
)
def _caller_may_probe_deployment(
deployment: Mapping[str, object], reachable_groups: frozenset[str], team_id: str | None, caller_is_admin: bool
deployment: Mapping[str, object], reachable: _ReachableDeployments, team_id: str | None, caller_is_admin: bool
) -> bool:
"""Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included."""
if not caller_is_admin and not Router.deployment_usable_by_team(deployment, team_id):
return False
return not reachable_groups.isdisjoint(_names_caller_may_request(deployment, team_id))
return reachable.include(deployment, team_id)
def _resolve_targeted_model_ids(
@ -1353,18 +1461,16 @@ async def health_endpoint(
detail={"error": "Model list not initialized"},
)
restrict_to_allowed_models: Final = not is_admin or not _key_models_unrestricted(user_api_key_dict)
reachable_groups: Final = (
await _model_groups_key_may_reach(
_DEPLOYMENT_LIST.validate_python(llm_model_list), user_api_key_dict, llm_router
)
reachable: Final = (
await _deployments_key_may_reach(_OBJECTS.validate_python(llm_model_list), user_api_key_dict, llm_router)
if restrict_to_allowed_models
else frozenset[str]()
else _NOTHING_REACHABLE
)
_llm_model_list: Final = [
m
for m in copy.deepcopy(llm_model_list)
if not restrict_to_allowed_models
or _caller_may_probe_deployment(m, reachable_groups, user_api_key_dict.team_id, is_admin)
or _caller_may_probe_deployment(m, reachable, user_api_key_dict.team_id, is_admin)
]
targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id)
if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids:

View file

@ -14694,9 +14694,11 @@ class Router:
"""
return [m for m in self.model_list if m["litellm_params"]["model"] == model]
def get_routable_deployments(self, model: str) -> tuple[DeploymentTypedDict, ...]:
def get_routable_deployments(
self, model: str, request_kwargs: Mapping[str, object] | None = None
) -> tuple[DeploymentTypedDict, ...]:
try:
_, candidates = self._common_checks_available_deployment(model=model, request_kwargs={}) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped
_, candidates = self._common_checks_available_deployment(model=model, request_kwargs=request_kwargs or {}) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # its signature leaves input and the result untyped
except (litellm.BadRequestError, ValueError):
return ()
if isinstance(candidates, dict):

View file

@ -4289,7 +4289,9 @@ _RESTRICTED_TEAM_KEY = UserAPIKeyAuth(
("caller", "expected_ids"),
[(_PROVIDER_WILDCARD_KEY, {"id-prefixed-nano", "id-mini"}), (_RESTRICTED_TEAM_KEY, {"id-mini"})],
)
async def test_health_endpoint_probes_the_deployments_auth_lets_the_key_request(caller, expected_ids):
async def test_health_endpoint_probes_the_deployments_auth_lets_the_key_request(
caller: UserAPIKeyAuth, expected_ids: set[str]
):
probed = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, caller)
assert probed == expected_ids
@ -4301,7 +4303,7 @@ async def test_health_endpoint_probes_the_deployments_auth_lets_the_key_request(
[(_PROVIDER_WILDCARD_KEY, ["id-prefixed-nano", "id-mini"]), (_RESTRICTED_TEAM_KEY, ["id-mini"])],
)
async def test_health_endpoint_returns_the_deployments_auth_lets_the_key_request_on_background_cache_path(
caller, expected_ids
caller: UserAPIKeyAuth, expected_ids: list[str]
):
from fastapi import Response
@ -4320,7 +4322,7 @@ async def test_health_endpoint_returns_the_deployments_auth_lets_the_key_request
@pytest.mark.asyncio
@pytest.mark.parametrize("use_background_health_checks", [False, True])
async def test_health_endpoint_refuses_a_deployment_the_team_allowlist_forbids(use_background_health_checks):
async def test_health_endpoint_refuses_a_deployment_the_team_allowlist_forbids(use_background_health_checks: bool):
from fastapi import HTTPException, Response
from litellm.proxy.health_endpoints._health_endpoints import health_endpoint
@ -4369,7 +4371,7 @@ def _router_with_aliases(model_group_alias: Mapping[str, str]) -> Router:
@pytest.mark.asyncio
@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES)
async def test_health_endpoint_probes_the_deployment_an_alias_sends_the_key_to_not_the_one_it_shadows(
caller, model_group_alias
caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str]
):
probed = await _live_probed_model_ids(
_PROVIDER_PREFIXED_MODEL_LIST, caller, llm_router=_router_with_aliases(model_group_alias)
@ -4381,7 +4383,7 @@ async def test_health_endpoint_probes_the_deployment_an_alias_sends_the_key_to_n
@pytest.mark.asyncio
@pytest.mark.parametrize(("caller", "model_group_alias"), _ALIAS_SHADOW_CASES)
async def test_health_endpoint_returns_the_deployment_an_alias_sends_the_key_to_on_background_cache_path(
caller, model_group_alias
caller: UserAPIKeyAuth, model_group_alias: Mapping[str, str]
):
from fastapi import Response
@ -4400,7 +4402,7 @@ async def test_health_endpoint_returns_the_deployment_an_alias_sends_the_key_to_
@pytest.mark.asyncio
@pytest.mark.parametrize("use_background_health_checks", [False, True])
async def test_health_endpoint_refuses_the_deployment_a_key_alias_shadows(use_background_health_checks):
async def test_health_endpoint_refuses_the_deployment_a_key_alias_shadows(use_background_health_checks: bool):
from fastapi import HTTPException, Response
from litellm.proxy.health_endpoints._health_endpoints import health_endpoint
@ -4438,6 +4440,102 @@ async def test_health_endpoint_never_probes_a_deployment_without_a_model_name_fo
assert probed == {"id-mini"}
@pytest.mark.asyncio
async def test_health_endpoint_never_probes_the_deployment_a_keys_router_alias_sends_it_to_but_forbids():
caller = UserAPIKeyAuth(
api_key="hashed-test-key",
models=["gpt-5.4-mini", "openai/gpt-5.4-nano"],
router_settings={"model_group_alias": {"gpt-5.4-mini": "claude-haiku-5-5"}},
)
probed = await _live_probed_model_ids(_PROVIDER_PREFIXED_MODEL_LIST, caller)
assert probed == {"id-prefixed-nano"}
_WILDCARD_DEPLOYMENT = {"model_name": "openai/*", "litellm_params": {"model": "openai/*"}, "model_info": {"id": "id-wild"}}
_CATCH_ALL_DEPLOYMENT = {"model_name": "*", "litellm_params": {"model": "*"}, "model_info": {"id": "id-star"}}
_TEAM_WILDCARD_DEPLOYMENT = {
"model_name": "openai-wild_team-a_7c1e",
"litellm_params": {"model": "openai/*"},
"model_info": {"id": "id-team-wild", "team_id": "team-a", "team_public_model_name": "openai/*"},
}
def _team_key(team_models: list[str], team_model_aliases: Mapping[str, str] | None = None) -> UserAPIKeyAuth:
return UserAPIKeyAuth(
api_key="hashed-test-key",
models=[],
team_id="team-a",
team_models=team_models,
team_model_aliases=dict(team_model_aliases) if team_model_aliases is not None else None,
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("model_list", "model_group_alias", "caller", "expected_ids"),
[
pytest.param(
[_WILDCARD_DEPLOYMENT, _PROVIDER_PREFIXED_MODEL_LIST[2]],
{},
_team_key(["openai/gpt-5.4-nano"]),
{"id-wild"},
id="team-name-served-by-provider-wildcard",
),
pytest.param(
[_TEAM_WILDCARD_DEPLOYMENT, _PROVIDER_PREFIXED_MODEL_LIST[2]],
{},
_team_key(["openai/gpt-5.4-nano"]),
{"id-team-wild"},
id="team-name-served-by-teams-own-wildcard",
),
pytest.param(
[_CATCH_ALL_DEPLOYMENT],
{},
_team_key(["gpt-5.4-mini"]),
{"id-star"},
id="team-name-served-by-catch-all",
),
pytest.param(
_PROVIDER_PREFIXED_MODEL_LIST,
{"gpt-4": "gpt-5.4-mini"},
_team_key(["gpt-4"]),
{"id-mini"},
id="team-allows-router-alias",
),
pytest.param(
_PROVIDER_PREFIXED_MODEL_LIST,
{},
_team_key(["my-alias"], {"my-alias": "gpt-5.4-mini"}),
{"id-mini"},
id="team-allows-team-alias",
),
pytest.param(
_PROVIDER_PREFIXED_MODEL_LIST,
{"claude-haiku-5-5": "gpt-5.4-mini"},
UserAPIKeyAuth(api_key="hashed-test-key", models=[]),
{"id-prefixed-nano", "id-mini", "id-haiku"},
id="unrestricted-key-calls-shadowed-deployment-by-id",
),
],
)
async def test_health_endpoint_probes_every_deployment_the_callers_allowed_names_route_to(
model_list: Sequence[Mapping[str, object]],
model_group_alias: Mapping[str, str],
caller: UserAPIKeyAuth,
expected_ids: set[str],
):
llm_router = Router(
model_list=copy.deepcopy([dict(deployment) for deployment in model_list]),
model_group_alias=dict(model_group_alias),
)
probed = await _live_probed_model_ids(model_list, caller, llm_router=llm_router)
assert probed == expected_ids
def test_resolve_targeted_model_ids_lets_model_id_win_over_model():
resolve = _health_endpoints_module._resolve_targeted_model_ids