Merge pull request #40765 from BerriAI/litellm_fix_health_scoped_results_and_serialization

fix(proxy): expand access groups in /health scoping and allowlist health display fields
This commit is contained in:
Mateo Wang 2026-09-11 21:21:44 -07:00 committed by GitHub
commit 98d46ee59d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 1220 additions and 119 deletions

View file

@ -32,32 +32,36 @@ from litellm.router_utils.auto_router_model_naming import (
strategy_router_dependencies,
)
ILLEGAL_DISPLAY_PARAMS: Final = [
"messages",
"api_key",
"prompt",
"input",
"client_secret",
"azure_ad_token",
"azure_username",
"azure_password",
"vertex_credentials",
"vertex_ai_credentials",
"aws_access_key_id",
"aws_secret_access_key",
"aws_session_token",
"aws_web_identity_token",
"extra_headers",
"headers",
"exception", # internal; not JSON-serializable, never for display
"litellm_metadata", # internal tracking metadata with auth objects; not for display
]
# Provider routing fields. Allowed for proxy admins so they can see which
# region/version a deployment is checking; gated at the endpoint layer for
# non-admin callers (see _strip_admin_only_fields_from_health_result).
ADMIN_ONLY_HEALTH_DISPLAY_PARAMS: Final = ("api_base", "api_version")
ADMIN_ONLY_HEALTH_DISPLAY_PARAMS: Final = ("api_base", "api_version", "aws_bedrock_runtime_endpoint")
MINIMAL_DISPLAY_PARAMS: Final = ["model", "mode_error"]
MINIMAL_DISPLAY_PARAMS: Final = frozenset({"model", "mode_error"})
HEALTH_DISPLAY_PARAMS: Final = (
MINIMAL_DISPLAY_PARAMS
| frozenset(ADMIN_ONLY_HEALTH_DISPLAY_PARAMS)
| frozenset(
{
"custom_llm_provider",
"mode",
"base_model",
"aws_region_name",
"region_name",
"watsonx_region_name",
"vertex_project",
"vertex_location",
"tpm",
"rpm",
"error",
"raw_request_typed_dict",
"x-ratelimit-remaining-requests",
"x-ratelimit-remaining-tokens",
"x-ms-region",
}
)
)
# Modes whose health-check probe is a chat-style completion call and
# therefore accept `max_tokens`. Other modes (embedding, image_generation,
@ -143,14 +147,10 @@ def _get_random_llm_message():
def _clean_endpoint_data(endpoint_data: dict, details: bool | None = True):
"""
Clean the endpoint data for display to users.
Keep only the explicitly approved, JSON-safe diagnostic fields for display to users.
"""
endpoint_data.pop("litellm_logging_obj", None)
return (
{k: v for k, v in endpoint_data.items() if k not in ILLEGAL_DISPLAY_PARAMS}
if details is not False
else {k: v for k, v in endpoint_data.items() if k in MINIMAL_DISPLAY_PARAMS}
)
displayed: Final = HEALTH_DISPLAY_PARAMS if details is not False else MINIMAL_DISPLAY_PARAMS
return {k: v for k, v in endpoint_data.items() if k in displayed}
def health_check_filter_kwargs_from_general_settings(
@ -258,8 +258,52 @@ def _deployment_model(deployment: Mapping[str, object]) -> str | None:
return params.get("model") if isinstance(params, Mapping) else None
def _owner_team_id(deployment: Mapping[str, object]) -> str | None:
info: Final = deployment.get("model_info")
owner: Final = info.get("team_id") if isinstance(info, Mapping) else None
return owner if isinstance(owner, str) else None
def _team_public_model_name(deployment: Mapping[str, object]) -> str | None:
info: Final = deployment.get("model_info")
name: Final = info.get("team_public_model_name") if isinstance(info, Mapping) else None
return name if isinstance(name, str) else None
def _deployments_routed_by_name(
model_list: Sequence[Mapping[str, object]], model_name: str, team_id: str | None
) -> tuple[Mapping[str, object], ...]:
"""The deployments a request for ``model_name`` from this caller routes to.
A team's own copies published under that name win, then deployments carrying it as
``model_name``. A caller with no team reaches a public name only when nothing carries
it as ``model_name``, and only an admin still has another team's deployment in a
scoped ``model_list`` by then.
"""
own_copies: Final = tuple(
x
for x in model_list
if team_id is not None and _owner_team_id(x) == team_id and _team_public_model_name(x) == model_name
)
if own_copies:
return own_copies
by_name: Final = tuple(x for x in model_list if x.get("model_name") == model_name)
if by_name or team_id is not None:
return by_name
return tuple(x for x in model_list if _team_public_model_name(x) == model_name)
def deployments_targeted_by_name(
model_list: Sequence[Mapping[str, object]], model: str, team_id: str | None
) -> tuple[Mapping[str, object], ...]:
"""``model`` targets deployments the way a request for it routes, else by ``litellm_params.model``."""
return _deployments_routed_by_name(model_list, model, team_id) or tuple(
x for x in model_list if _deployment_model(x) == model
)
def _narrow_to_target(
model_list: Sequence[Mapping[str, object]], model: str | None, model_id: str | None
model_list: Sequence[Mapping[str, object]], model: str | None, model_id: str | None, team_id: str | None
) -> tuple[Mapping[str, object], ...]:
"""Narrow to the requested deployment. An id matching nothing keeps the whole list."""
if model_id is not None:
@ -267,8 +311,7 @@ def _narrow_to_target(
return by_id or tuple(model_list)
if model is None:
return tuple(model_list)
by_param: Final = tuple(x for x in model_list if _deployment_model(x) == model)
return by_param or tuple(x for x in model_list if x.get("model_name") == model)
return deployments_targeted_by_name(model_list, model, team_id)
def _is_strategy_router_deployment(litellm_params: Mapping[str, object]) -> bool:
@ -813,13 +856,18 @@ async def perform_health_check(
instrumentation_context: dict | None = None,
health_check_skip_disabled_background_models: bool = False,
router: "Router | None" = None,
team_id: str | None = None,
):
"""
Perform a health check on the system.
When model_id is provided, only the deployment with that id is checked
(so models that share the same name but have different ids are checked separately).
When model (name) is provided, all deployments matching that name are checked.
When model (name) is provided, the deployments a request for that name from the
caller (``team_id``) would route to are checked: the caller's team copies published
under that name, else the deployments named that way, else a public name that only
another team's deployment carries, else the deployments whose ``litellm_params.model``
is that string.
When ``health_check_skip_disabled_background_models`` is True (via
``general_settings.health_check_skip_disabled_background_models``), deployments
@ -850,7 +898,7 @@ async def perform_health_check(
cycle_start_time: Final = time.monotonic()
requested_model_count: Final = len(model_list)
skip_disabled: Final = health_check_skip_disabled_background_models
narrowed: Final = _health_check_eligible(_narrow_to_target(model_list, model, model_id), skip_disabled)
narrowed: Final = _health_check_eligible(_narrow_to_target(model_list, model, model_id, team_id), skip_disabled)
if not narrowed:
if instrumentation_enabled:
logger.debug(

View file

@ -36,9 +36,13 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
WebhookEvent,
)
from litellm.proxy.auth.auth_checks import (
_resolve_key_models_for_auth_check, # pyright: ignore[reportPrivateUsage] # the auth layer's sentinel resolution, reused so /health scopes exactly like a request
)
from litellm.proxy.auth.auth_utils import (
_BANNED_REQUEST_BODY_PARAMS, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the request-body check
)
from litellm.proxy.auth.model_checks import get_key_models
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.proxy.db.health_check_latest import LatestHealthCheckRow
@ -47,6 +51,7 @@ from litellm.proxy.health_check import (
ADMIN_ONLY_HEALTH_DISPLAY_PARAMS,
_clean_endpoint_data,
_update_litellm_params_for_health_check,
deployments_targeted_by_name,
health_check_filter_kwargs_from_general_settings,
perform_health_check,
run_with_timeout,
@ -58,6 +63,7 @@ from litellm.proxy.middleware.in_flight_requests_middleware import (
get_in_flight_requests,
)
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
from litellm.router import Router
from litellm.router_utils.clientside_credential_handler import (
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path
clientside_credential_keys,
@ -917,7 +923,7 @@ def _is_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> bool:
def _strip_admin_only_fields_from_health_result(result: dict) -> dict:
"""
Return a copy of the /health response with provider routing fields
(``api_base``, ``api_version``) removed from each healthy/unhealthy
(``ADMIN_ONLY_HEALTH_DISPLAY_PARAMS``) removed from each healthy/unhealthy
endpoint entry. Used to hide those fields from non-admin callers while
still showing them which deployments they own and whether each one is
healthy. Proxy admins receive the unmodified result.
@ -931,41 +937,68 @@ def _strip_admin_only_fields_from_health_result(result: dict) -> dict:
return out
def _resolve_targeted_model_ids(model_list: list, model: str | None, model_id: str | None) -> set | None:
def _health_accessible_model_names(
user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None
) -> frozenset[str] | None:
"""Model names the caller may health-check, or None when the key is unrestricted."""
granted_models: Final = _resolve_key_models_for_auth_check(user_api_key_dict)
if not granted_models or SpecialModelNames.all_proxy_models.value in granted_models:
return None
if llm_router is None:
return frozenset(granted_models)
return frozenset(
get_key_models(
user_api_key_dict=user_api_key_dict,
proxy_model_list=llm_router.get_model_names(team_id=user_api_key_dict.team_id),
model_access_groups=llm_router.get_model_access_groups(),
)
)
def _caller_may_probe_deployment(
deployment: Mapping[str, object],
allowed_models: frozenset[str] | None,
llm_router: Router | None,
team_id: str | None,
caller_is_admin: bool,
) -> bool:
"""Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included."""
if not caller_is_admin and not Router._deployment_usable_by_team(deployment, team_id):
return False
if allowed_models is None:
return True
if llm_router is None:
return deployment.get("model_name") in allowed_models
model: Final = dict(deployment)
return any(
llm_router.should_include_deployment(model_name=name, model=model, team_id=team_id) for name in allowed_models
)
def _resolve_targeted_model_ids(
model_list: list, model: str | None, model_id: str | None, team_id: str | None
) -> set | None:
"""
Resolve a ``/health`` ``model`` / ``model_id`` query param to the set of
deployment IDs the response should be scoped to.
deployment IDs the response should be scoped to, mirroring the live-path
narrowing in ``perform_health_check()``: ``model_id`` wins when given and
matches ``model_info.id`` only; ``model`` targets the deployments a request
for that name from the caller would route to, else those whose
``litellm_params.model`` provider string is that value (``deployments_targeted_by_name``).
Mirrors the live-path semantics in ``perform_health_check()``: ``model``
matches either the deployment's ``model_name`` alias or its
``litellm_params.model`` provider string. ``model_id`` matches
``model_info.id``.
Both query params are validated against the supplied ``model_list``.
Callers pass an already-scoped list (filtered to the caller's allowed
models for non-admins, full list for admins), so a ``model_id`` that
isn't present resolves to an empty set rather than a single-element
set preventing a non-admin from reading another deployment's cached
health entry by guessing its ID.
Returns ``None`` when no targeting is requested callers should treat
that as "no filter."
Callers pass an already-scoped list, so a ``model_id`` outside the
caller's scope resolves to an empty set and never to the unvalidated id.
Returns ``None`` when no targeting is requested.
"""
if not model and not model_id:
if model_id:
return {i for m in model_list if (i := (m.get("model_info") or {}).get("id")) == model_id}
if not model:
return None
target_ids: Final[set] = set()
for m in model_list:
deployment_id = (m.get("model_info") or {}).get("id")
if not deployment_id:
continue
if model_id and deployment_id == model_id:
target_ids.add(deployment_id)
continue
if model:
litellm_model = (m.get("litellm_params") or {}).get("model")
if m.get("model_name") == model or litellm_model == model:
target_ids.add(deployment_id)
return target_ids
return {
i
for m in deployments_targeted_by_name(model_list, model, team_id)
if (i := (m.get("model_info") or {}).get("id"))
}
def _filter_health_check_results_by_model_ids(results: dict, allowed_model_ids: set) -> dict:
@ -1046,8 +1079,12 @@ def _health_endpoint_resolve_target_model_name(
model_id: str | None,
llm_router,
) -> str | None:
"""Map ``model_id`` (without ``model``) to ``model_name`` for live health checks."""
if not model_id or model:
"""Map ``model_id`` to its deployment's ``model_name`` for live health checks.
``model_id`` wins over ``model``, so an id no deployment carries is a 404 even
when it is paired with a known name.
"""
if not model_id:
return model
if llm_router is None:
raise HTTPException(
@ -1133,7 +1170,9 @@ async def health_endpoint(
response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
if is_admin:
return result
response.headers["Litellm-Health-Field-Notice"] = "api_base and api_version are admin-only on this endpoint"
response.headers["Litellm-Health-Field-Notice"] = (
f"{', '.join(ADMIN_ONLY_HEALTH_DISPLAY_PARAMS)} are admin-only on this endpoint"
)
return _strip_admin_only_fields_from_health_result(result)
try:
@ -1157,32 +1196,24 @@ async def health_endpoint(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={"error": "Model list not initialized"},
)
_llm_model_list = copy.deepcopy(llm_model_list)
### FILTER MODELS FOR ONLY THOSE USER HAS ACCESS TO ###
# Live path: scope by model_name (every deployment has one).
# Cache path: scope by model_id (the cache is keyed on model_id).
# Consequence: a deployment whose model_name the caller can access
# but which lacks model_info.id will appear in the live /health
# response but NOT in the background-cache /health response. This is
# surfaced via the "warnings" field below so operators can fix the
# missing model_info.id rather than guess at the discrepancy.
# Keys granted SpecialModelNames.all_proxy_models carry the literal
# "all-proxy-models" entry, which matches no real model_name; treat
# them as unrestricted instead of filtering the list down to nothing.
# Keys granted SpecialModelNames.all_team_models inherit the parent
# team's allowlist (same semantics as get_key_models in
# model_checks.py). Without a team_id the sentinel cannot resolve and
# stays in the list, matching nothing; denied rather than
# unrestricted, mirroring _resolve_key_models_for_auth_check.
accessible_models = list(user_api_key_dict.models)
if SpecialModelNames.all_team_models.value in accessible_models and user_api_key_dict.team_id is not None:
accessible_models = list(user_api_key_dict.team_models)
restrict_to_allowed_models: Final = (
len(accessible_models) > 0 and SpecialModelNames.all_proxy_models.value not in accessible_models
)
if restrict_to_allowed_models:
allowed_models: Final = set(accessible_models)
_llm_model_list = [m for m in _llm_model_list if m.get("model_name") in allowed_models]
allowed_models: Final = _health_accessible_model_names(user_api_key_dict, llm_router)
restrict_to_allowed_models: Final = not is_admin or allowed_models is not None
_llm_model_list: Final = [
m
for m in copy.deepcopy(llm_model_list)
if not restrict_to_allowed_models
or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id, is_admin)
]
targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id)
if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail={
"error": f"key not allowed to health-check model_id {model_id}"
if model_id
else f"key not allowed to health-check model {model}"
},
)
if use_background_health_checks:
# The cached background result covers every model. When the
# caller targets a specific model/model_id we have to narrow the
@ -1190,7 +1221,6 @@ async def health_endpoint(
# healthy_count, otherwise an unhealthy "foo" combined with any
# other healthy model would still report healthy_count > 0 and
# the targeted-503 path would never fire.
targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id)
if restrict_to_allowed_models:
allowed_model_ids: Final = {
(m.get("model_info") or {}).get("id")
@ -1202,7 +1232,7 @@ async def health_endpoint(
# intersection of "targeted" and "allowed."
filter_ids: Final = targeted_ids if targeted_ids is not None else allowed_model_ids
filtered: Final = _filter_health_check_results_by_model_ids(health_check_results, filter_ids)
if targeted_ids is None and not allowed_model_ids:
if targeted_ids is None and _llm_model_list and not allowed_model_ids:
# Caller has accessible model_names but none of the
# matching deployments expose a model_info.id, so the
# cache filter (which keys on model_id) drops every
@ -1241,6 +1271,7 @@ async def health_endpoint(
model_id=model_id,
max_concurrency=health_check_concurrency,
router=llm_router,
team_id=user_api_key_dict.team_id,
**_hc_filter,
)
return _post_process(router_result)

View file

@ -735,6 +735,121 @@ async def test_perform_health_check_and_save_forwards_skip_disabled_background_f
assert call_kwargs["health_check_skip_disabled_background_models"] is True
@pytest.mark.asyncio
async def test_perform_health_check_narrows_to_a_team_deployment_by_its_public_name():
"""``/health?model=<team_public_model_name>`` must probe the team deployment, not an empty list."""
from litellm.proxy.health_check import perform_health_check
team_deployment = {
"model_name": "bedrock-nova_team-b_9f2c",
"litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"},
"model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"},
}
other_deployment = {
"model_name": "gpt-5.4-mini",
"litellm_params": {"model": "openai/gpt-5.4-mini"},
"model_info": {"id": "id-openai"},
}
probe = AsyncMock(return_value=([{"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-team-b"}], [], {}))
with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam
"litellm.proxy.health_check._perform_health_check", probe
):
healthy, unhealthy, _ = await perform_health_check(
model_list=[team_deployment, other_deployment], model="bedrock-nova", team_id="team-b"
)
assert [m["model_info"]["id"] for m in probe.call_args.args[0]] == ["id-team-b"]
assert [ep["model_id"] for ep in healthy] == ["id-team-b"]
assert unhealthy == []
@pytest.mark.asyncio
async def test_perform_health_check_keeps_a_public_name_off_another_team():
"""A team's public model name is not a global alias: a caller from another team must not probe its deployment."""
from litellm.proxy.health_check import perform_health_check
team_deployment = {
"model_name": "bedrock-nova_team-b_9f2c",
"litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"},
"model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"},
}
probe = AsyncMock(return_value=([{"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-team-b"}], [], {}))
with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam
"litellm.proxy.health_check._perform_health_check", probe
):
healthy, unhealthy, _ = await perform_health_check(
model_list=[team_deployment], model="bedrock-nova", team_id="team-a"
)
probe.assert_not_awaited()
assert healthy == []
assert unhealthy == []
_GLOBAL_DEPLOYMENT = {
"model_name": "bedrock-nova",
"litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"},
"model_info": {"id": "id-bedrock"},
}
_TEAM_B_COPY = {
"model_name": "bedrock-nova_team-b_9f2c",
"litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"},
"model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"},
}
_GLOBAL_BARE_NAME = {
"model_name": "gpt-5.4-nano",
"litellm_params": {"model": "gpt-5.4-nano"},
"model_info": {"id": "id-nano"},
}
_TEAM_B_BARE_COPY = {
"model_name": "gpt-5.4-nano_team-b_7c3d",
"litellm_params": {"model": "gpt-5.4-nano"},
"model_info": {"id": "id-nano-team-b", "team_id": "team-b", "team_public_model_name": "gpt-5.4-nano"},
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("team_id", "model", "model_list", "expected_ids"),
[
(None, "bedrock-nova", [_TEAM_B_COPY], ["id-team-b"]),
(None, "bedrock-nova", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock"]),
("team-b", "bedrock-nova", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-team-b"]),
("team-b", "gpt-5.4-nano", [_GLOBAL_BARE_NAME, _TEAM_B_BARE_COPY], ["id-nano-team-b"]),
(None, "gpt-5.4-nano", [_GLOBAL_BARE_NAME, _TEAM_B_BARE_COPY], ["id-nano"]),
(None, "bedrock/us.amazon.nova-2-lite-v1:0", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock", "id-team-b"]),
],
ids=[
"a team-less caller reaches a public name nothing else carries",
"model_name wins over a public name for a team-less caller",
"a team's own copy wins over the global model_name",
"a team's own copy wins over a litellm_params.model equal to the public name",
"model_name wins over a litellm_params.model equal to it for a team-less caller",
"a provider model string no name carries still matches litellm_params.model",
],
)
async def test_perform_health_check_targets_a_name_the_way_a_request_for_it_routes(
team_id, model, model_list, expected_ids
):
"""``/health?model=<name>`` probes the deployments a request for that name from the same caller would route to."""
from litellm.proxy.health_check import perform_health_check
probe = AsyncMock(
return_value=([{"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": i} for i in expected_ids], [], {})
)
with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam
"litellm.proxy.health_check._perform_health_check", probe
):
healthy, unhealthy, _ = await perform_health_check(model_list=model_list, model=model, team_id=team_id)
assert [m["model_info"]["id"] for m in probe.call_args.args[0]] == expected_ids
assert [ep["model_id"] for ep in healthy] == expected_ids
assert unhealthy == []
def test_parse_background_health_check_model_groups_unset_returns_none():
from litellm.proxy.health_check import parse_background_health_check_model_groups

View file

@ -872,9 +872,9 @@ def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list():
"""Pinned because the disabled-dependency fix moved this filter into its own helper."""
deployments = [{"model_name": "a", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a-1"}}]
assert hc_module._narrow_to_target(deployments, None, "no-such-id") == tuple(deployments)
assert hc_module._narrow_to_target(deployments, None, "a-1") == tuple(deployments)
assert hc_module._narrow_to_target(deployments, "a", None) == tuple(deployments)
assert hc_module._narrow_to_target(deployments, None, "no-such-id", None) == tuple(deployments)
assert hc_module._narrow_to_target(deployments, None, "a-1", None) == tuple(deployments)
assert hc_module._narrow_to_target(deployments, "a", None, None) == tuple(deployments)
def _nested_router_fixture(parent_tier: str):