From e75dd5663117026c3b416695b7eafaf3b5cfe1da Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:24:42 -0700 Subject: [PATCH 1/9] fix(proxy): expand access groups in /health scoping and allowlist health display fields /health filtered deployments by the key's literal `models` entries, so a key or UI user scoped to a model access group got 0/0 on both the live and the background-cache path, plus a misleading "missing model_info.id" warning on the cached one. The endpoint now resolves key sentinels and expands access groups the way auth does. Health entries used to copy every litellm_params field, so a deployment parameter that is not JSON-safe (a nested mapping keyed by a tuple) 500d the endpoint for every caller and admins saw internal settings nobody asked for. Entries now keep only an explicit allowlist of JSON-safe diagnostic fields; api_base and api_version stay admin-only and credentials stay out for everyone. Fixes #28206 --- litellm/proxy/health_check.py | 55 ++-- .../health_endpoints/_health_endpoints.py | 56 ++-- .../health_endpoints/test_health_endpoints.py | 286 ++++++++++++++++++ 3 files changed, 342 insertions(+), 55 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 219f6f270ed..8daeb296f3e 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -32,32 +32,35 @@ from litellm.router_utils.auto_router_model_naming import ( strategy_router_dependencies, ) -ILLEGAL_DISPLAY_PARAMS: Final = [ - "messages", - "api_key", - "prompt", - "input", - "client_secret", - "azure_ad_token", - "azure_username", - "azure_password", - "vertex_credentials", - "vertex_ai_credentials", - "aws_access_key_id", - "aws_secret_access_key", - "aws_session_token", - "aws_web_identity_token", - "extra_headers", - "headers", - "exception", # internal; not JSON-serializable, never for display - "litellm_metadata", # internal tracking metadata with auth objects; not for display -] # Provider routing fields. Allowed for proxy admins so they can see which # region/version a deployment is checking; gated at the endpoint layer for # non-admin callers (see _strip_admin_only_fields_from_health_result). ADMIN_ONLY_HEALTH_DISPLAY_PARAMS: Final = ("api_base", "api_version") -MINIMAL_DISPLAY_PARAMS: Final = ["model", "mode_error"] +MINIMAL_DISPLAY_PARAMS: Final = frozenset({"model", "mode_error"}) + +HEALTH_DISPLAY_PARAMS: Final = ( + MINIMAL_DISPLAY_PARAMS + | frozenset(ADMIN_ONLY_HEALTH_DISPLAY_PARAMS) + | frozenset( + { + "custom_llm_provider", + "mode", + "base_model", + "aws_region_name", + "region_name", + "vertex_project", + "vertex_location", + "tpm", + "rpm", + "error", + "raw_request_typed_dict", + "x-ratelimit-remaining-requests", + "x-ratelimit-remaining-tokens", + "x-ms-region", + } + ) +) # Modes whose health-check probe is a chat-style completion call and # therefore accept `max_tokens`. Other modes (embedding, image_generation, @@ -143,14 +146,10 @@ def _get_random_llm_message(): def _clean_endpoint_data(endpoint_data: dict, details: bool | None = True): """ - Clean the endpoint data for display to users. + Keep only the explicitly approved, JSON-safe diagnostic fields for display to users. """ - endpoint_data.pop("litellm_logging_obj", None) - return ( - {k: v for k, v in endpoint_data.items() if k not in ILLEGAL_DISPLAY_PARAMS} - if details is not False - else {k: v for k, v in endpoint_data.items() if k in MINIMAL_DISPLAY_PARAMS} - ) + displayed: Final = HEALTH_DISPLAY_PARAMS if details is not False else MINIMAL_DISPLAY_PARAMS + return {k: v for k, v in endpoint_data.items() if k in displayed} def health_check_filter_kwargs_from_general_settings( diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 8b57bdca2fe..0810bd2b095 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -36,9 +36,13 @@ from litellm.proxy._types import ( UserAPIKeyAuth, WebhookEvent, ) +from litellm.proxy.auth.auth_checks import ( + _resolve_key_models_for_auth_check, # pyright: ignore[reportPrivateUsage] # the auth layer's sentinel resolution, reused so /health scopes exactly like a request +) from litellm.proxy.auth.auth_utils import ( _BANNED_REQUEST_BODY_PARAMS, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the request-body check ) +from litellm.proxy.auth.model_checks import get_key_models from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler from litellm.proxy.db.proxy_worker_heartbeat import count_live_proxy_workers @@ -54,6 +58,7 @@ from litellm.proxy.middleware.in_flight_requests_middleware import ( get_in_flight_requests, ) from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager +from litellm.router import Router from litellm.router_utils.clientside_credential_handler import ( _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path clientside_credential_keys, @@ -854,6 +859,24 @@ def _strip_admin_only_fields_from_health_result(result: dict) -> dict: return out +def _health_accessible_model_names( + user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None +) -> frozenset[str] | None: + """Model names the caller may health-check, or None when the key is unrestricted.""" + granted_models: Final = _resolve_key_models_for_auth_check(user_api_key_dict) + if not granted_models or SpecialModelNames.all_proxy_models.value in granted_models: + return None + if llm_router is None: + return frozenset(granted_models) + return frozenset( + get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=llm_router.get_model_names(team_id=user_api_key_dict.team_id), + model_access_groups=llm_router.get_model_access_groups(), + ) + ) + + def _resolve_targeted_model_ids(model_list: list, model: str | None, model_id: str | None) -> set | None: """ Resolve a ``/health`` ``model`` / ``model_id`` query param to the set of @@ -1080,32 +1103,11 @@ async def health_endpoint( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail={"error": "Model list not initialized"}, ) - _llm_model_list = copy.deepcopy(llm_model_list) - ### FILTER MODELS FOR ONLY THOSE USER HAS ACCESS TO ### - # Live path: scope by model_name (every deployment has one). - # Cache path: scope by model_id (the cache is keyed on model_id). - # Consequence: a deployment whose model_name the caller can access - # but which lacks model_info.id will appear in the live /health - # response but NOT in the background-cache /health response. This is - # surfaced via the "warnings" field below so operators can fix the - # missing model_info.id rather than guess at the discrepancy. - # Keys granted SpecialModelNames.all_proxy_models carry the literal - # "all-proxy-models" entry, which matches no real model_name; treat - # them as unrestricted instead of filtering the list down to nothing. - # Keys granted SpecialModelNames.all_team_models inherit the parent - # team's allowlist (same semantics as get_key_models in - # model_checks.py). Without a team_id the sentinel cannot resolve and - # stays in the list, matching nothing; denied rather than - # unrestricted, mirroring _resolve_key_models_for_auth_check. - accessible_models = list(user_api_key_dict.models) - if SpecialModelNames.all_team_models.value in accessible_models and user_api_key_dict.team_id is not None: - accessible_models = list(user_api_key_dict.team_models) - restrict_to_allowed_models: Final = ( - len(accessible_models) > 0 and SpecialModelNames.all_proxy_models.value not in accessible_models - ) - if restrict_to_allowed_models: - allowed_models: Final = set(accessible_models) - _llm_model_list = [m for m in _llm_model_list if m.get("model_name") in allowed_models] + allowed_models: Final = _health_accessible_model_names(user_api_key_dict, llm_router) + restrict_to_allowed_models: Final = allowed_models is not None + _llm_model_list: Final = [ + m for m in copy.deepcopy(llm_model_list) if allowed_models is None or m.get("model_name") in allowed_models + ] if use_background_health_checks: # The cached background result covers every model. When the # caller targets a specific model/model_id we have to narrow the @@ -1125,7 +1127,7 @@ async def health_endpoint( # intersection of "targeted" and "allowed." filter_ids: Final = targeted_ids if targeted_ids is not None else allowed_model_ids filtered: Final = _filter_health_check_results_by_model_ids(health_check_results, filter_ids) - if targeted_ids is None and not allowed_model_ids: + if targeted_ids is None and _llm_model_list and not allowed_model_ids: # Caller has accessible model_names but none of the # matching deployments expose a model_info.id, so the # cache filter (which keys on model_id) drops every diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index e3f71692c78..b14baf5f8c5 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -1,6 +1,8 @@ import asyncio import json import time +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager from datetime import datetime, timedelta from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch @@ -1650,6 +1652,209 @@ async def test_health_endpoint_resolves_all_team_models_to_team_allowlist(): assert returned_names == {"model-b"}, f"all-team-models key should health-check the team's models: {returned_names}" +class _AccessGroupRouter: + """Router stand-in exposing only the two lookups /health uses to expand a key's model grants.""" + + def __init__(self, access_groups: dict[str, list[str]], model_names: list[str]) -> None: + self._access_groups = access_groups + self._model_names = model_names + + def get_model_access_groups(self, model_name=None, model_access_group=None, team_id=None): + return self._access_groups + + def get_model_names(self, team_id=None): + return self._model_names + + +_ACCESS_GROUP_MODEL_LIST = [ + { + "model_name": "bedrock-nova", + "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, + "model_info": {"id": "id-bedrock", "access_groups": ["bedrock-group"]}, + }, + { + "model_name": "gpt-5.4-mini", + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "id-openai"}, + }, +] +_ACCESS_GROUP_ROUTER = _AccessGroupRouter({"bedrock-group": ["bedrock-nova"]}, ["bedrock-nova", "gpt-5.4-mini"]) +_ACCESS_GROUP_CACHED_RESULTS = { + "healthy_endpoints": [ + {"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-bedrock"}, + {"model": "openai/gpt-5.4-mini", "model_id": "id-openai"}, + ], + "unhealthy_endpoints": [], + "healthy_count": 2, + "unhealthy_count": 0, +} + + +@contextmanager +def _proxy_health_globals( + llm_model_list: Sequence[Mapping[str, object]], + llm_router: object, + use_background_health_checks: bool = False, + health_check_results: Mapping[str, object] | None = None, +) -> Iterator[None]: + with ( + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.llm_model_list", list(llm_model_list) + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.llm_router", llm_router + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.prisma_client", None + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.use_background_health_checks", use_background_health_checks + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.user_model", None + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.health_check_results", dict(health_check_results or {}) + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.health_check_details", True + ), + patch( # test-quality-ok: proxy module global, no injection seam + "litellm.proxy.proxy_server.health_check_concurrency", 1 + ), + ): + yield + + +@pytest.mark.asyncio +async def test_health_endpoint_expands_access_group_on_live_path(): + """ + LIT-6907 / gh-28206: a key granted a model access group carries the group + name in user_api_key_dict.models. Matching it as a literal model_name + filtered every deployment out and /health answered 0/0 for a model the + same key could call. + """ + from fastapi import Response + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + captured: dict = {} + + async def fake_perform(**kwargs): + captured["model_list"] = kwargs["model_list"] + return {"healthy_endpoints": [], "unhealthy_endpoints": [], "healthy_count": 0, "unhealthy_count": 0} + + with ( + _proxy_health_globals(_ACCESS_GROUP_MODEL_LIST, _ACCESS_GROUP_ROUTER), + patch( # test-quality-ok: the model list handed to the probe is the assertion; no injection seam + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", + side_effect=fake_perform, + ), + ): + await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"]), + ) + + assert [m["model_name"] for m in captured["model_list"]] == ["bedrock-nova"] + + +@pytest.mark.asyncio +async def test_health_endpoint_expands_access_group_on_background_cache_path(): + """ + LIT-6907: the background-cache path scoped the cached entries through the + same literal model_name match, so an access-group key got an empty result + plus a warning blaming missing model_info.id. + """ + from fastapi import Response + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _ACCESS_GROUP_MODEL_LIST, + _ACCESS_GROUP_ROUTER, + use_background_health_checks=True, + health_check_results=_ACCESS_GROUP_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"]), + model=None, + model_id=None, + ) + + assert [e["model_id"] for e in result["healthy_endpoints"]] == ["id-bedrock"] + assert result["healthy_count"] == 1 + assert "warnings" not in result + + +@pytest.mark.asyncio +async def test_health_endpoint_treats_no_team_all_team_models_as_unrestricted(): + """ + A key granted "all-team-models" without a team resolves to an empty + allowlist in the auth layer, which means unrestricted. /health used to + keep the unresolved sentinel and filter every deployment out instead. + """ + from fastapi import Response + + from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + captured: dict = {} + + async def fake_perform(**kwargs): + captured["model_list"] = kwargs["model_list"] + return {"healthy_endpoints": [], "unhealthy_endpoints": [], "healthy_count": 0, "unhealthy_count": 0} + + with ( + _proxy_health_globals(_ACCESS_GROUP_MODEL_LIST, None), + patch( # test-quality-ok: the model list handed to the probe is the assertion; no injection seam + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", + side_effect=fake_perform, + ), + ): + await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth( + api_key="hashed-test-key", models=[SpecialModelNames.all_team_models.value], team_id=None + ), + ) + + assert {m["model_name"] for m in captured["model_list"]} == {"bedrock-nova", "gpt-5.4-mini"} + + +@pytest.mark.asyncio +async def test_health_endpoint_omits_model_id_warning_when_no_deployment_matches(): + """ + The missing-model_info.id warning is only true when a matching deployment + exists without an id. A key whose grants match no deployment at all gets a + plain empty result, not advice to populate ids that are already there. + """ + from fastapi import Response + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _ACCESS_GROUP_MODEL_LIST, + _ACCESS_GROUP_ROUTER, + use_background_health_checks=True, + health_check_results=_ACCESS_GROUP_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["no-such-model"]), + model=None, + model_id=None, + ) + + assert result["healthy_count"] == 0 + assert result["unhealthy_count"] == 0 + assert "warnings" not in result + + @pytest.mark.asyncio async def test_health_endpoint_filters_background_cache_by_user_access(): """ @@ -2571,6 +2776,87 @@ def test_clean_endpoint_data_never_displays_credential_fields(credential_field, assert canary not in str(cleaned) +def test_clean_endpoint_data_keeps_only_json_safe_diagnostics(): + """ + LIT-6907: _clean_endpoint_data used to copy every litellm_param not on a + deny list, so a nested mapping keyed by a tuple reached jsonable_encoder + and 500'd /health. Only the explicit allowlist survives now. + """ + from fastapi.encoders import jsonable_encoder + + from litellm.proxy.health_check import _clean_endpoint_data + + cleaned = _clean_endpoint_data( + { + "model": "bedrock/us.amazon.nova-2-lite-v1:0", + "custom_llm_provider": "bedrock", + "aws_region_name": "us-east-1", + "metadata": {("us-east-1", "primary"): "canary-nested-mapping"}, + "allow_client_keepalive_override": False, + "api_key": "CANARY-API-KEY", + "x-ratelimit-remaining-requests": 99, + "raw_request_typed_dict": {"raw_request_api_base": "https://example.test"}, + }, + details=True, + ) + + assert cleaned == { + "model": "bedrock/us.amazon.nova-2-lite-v1:0", + "custom_llm_provider": "bedrock", + "aws_region_name": "us-east-1", + "x-ratelimit-remaining-requests": 99, + "raw_request_typed_dict": {"raw_request_api_base": "https://example.test"}, + } + assert jsonable_encoder(cleaned) == cleaned + + +@pytest.mark.asyncio +async def test_health_endpoint_result_survives_non_json_safe_deployment_params(): + """ + LIT-6907: the full /health path with a deployment carrying a tuple-keyed + nested mapping must produce a response FastAPI can encode, with the + approved diagnostics intact and the offending param absent. + """ + from fastapi import Response + from fastapi.encoders import jsonable_encoder + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + model_list = [ + { + "model_name": "bedrock-nova", + "litellm_params": { + "model": "bedrock/us.amazon.nova-2-lite-v1:0", + "aws_region_name": "us-east-1", + "aws_access_key_id": "CANARY-ACCESS-KEY", + "metadata": {("us-east-1", "primary"): "canary-nested-mapping"}, + }, + "model_info": {"id": "id-bedrock"}, + } + ] + + with ( + _proxy_health_globals(model_list, None), + patch( # test-quality-ok: the provider probe is faked; the assertion is the response shaping after it + "litellm.ahealth_check", AsyncMock(return_value={"x-ratelimit-remaining-requests": 99}) + ), + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-admin-key", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + encoded = jsonable_encoder(result) + assert encoded["healthy_count"] == 1 + entry = encoded["healthy_endpoints"][0] + assert entry["model_id"] == "id-bedrock" + assert entry["aws_region_name"] == "us-east-1" + assert entry["x-ratelimit-remaining-requests"] == 99 + assert "metadata" not in entry + assert "CANARY" not in str(encoded) + + class TestConfigBaseForHealthCheck: """A request that sets its own connection fields gets a base without the configuration's credentials; anything it leaves unset still comes from From a84f4d6206cfdfa8b025421b4bbdf43a55eed5ee Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:57:13 -0700 Subject: [PATCH 2/9] fix(proxy): keep /health inside the caller's team and admin-gate the Bedrock runtime endpoint --- litellm/proxy/health_check.py | 3 +- .../health_endpoints/_health_endpoints.py | 23 ++- .../health_endpoints/test_health_endpoints.py | 161 ++++++++++++++++-- 3 files changed, 169 insertions(+), 18 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 8daeb296f3e..78bb01aa6ac 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -35,7 +35,7 @@ from litellm.router_utils.auto_router_model_naming import ( # Provider routing fields. Allowed for proxy admins so they can see which # region/version a deployment is checking; gated at the endpoint layer for # non-admin callers (see _strip_admin_only_fields_from_health_result). -ADMIN_ONLY_HEALTH_DISPLAY_PARAMS: Final = ("api_base", "api_version") +ADMIN_ONLY_HEALTH_DISPLAY_PARAMS: Final = ("api_base", "api_version", "aws_bedrock_runtime_endpoint") MINIMAL_DISPLAY_PARAMS: Final = frozenset({"model", "mode_error"}) @@ -49,6 +49,7 @@ HEALTH_DISPLAY_PARAMS: Final = ( "base_model", "aws_region_name", "region_name", + "watsonx_region_name", "vertex_project", "vertex_location", "tpm", diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 0810bd2b095..cc9ad1eb0cb 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -845,7 +845,7 @@ def _is_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> bool: def _strip_admin_only_fields_from_health_result(result: dict) -> dict: """ Return a copy of the /health response with provider routing fields - (``api_base``, ``api_version``) removed from each healthy/unhealthy + (``ADMIN_ONLY_HEALTH_DISPLAY_PARAMS``) removed from each healthy/unhealthy endpoint entry. Used to hide those fields from non-admin callers while still showing them which deployments they own and whether each one is healthy. Proxy admins receive the unmodified result. @@ -877,6 +877,18 @@ def _health_accessible_model_names( ) +def _caller_may_probe_deployment( + deployment: Mapping[str, object], allowed_models: frozenset[str], llm_router: Router | None, team_id: str | None +) -> bool: + """Same deployment visibility rule as request auth: another team's deployment is never in scope.""" + if llm_router is None: + return deployment.get("model_name") in allowed_models + model: Final = dict(deployment) + return any( + llm_router.should_include_deployment(model_name=name, model=model, team_id=team_id) for name in allowed_models + ) + + def _resolve_targeted_model_ids(model_list: list, model: str | None, model_id: str | None) -> set | None: """ Resolve a ``/health`` ``model`` / ``model_id`` query param to the set of @@ -1079,7 +1091,9 @@ async def health_endpoint( response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE if is_admin: return result - response.headers["Litellm-Health-Field-Notice"] = "api_base and api_version are admin-only on this endpoint" + response.headers["Litellm-Health-Field-Notice"] = ( + f"{', '.join(ADMIN_ONLY_HEALTH_DISPLAY_PARAMS)} are admin-only on this endpoint" + ) return _strip_admin_only_fields_from_health_result(result) try: @@ -1106,7 +1120,10 @@ async def health_endpoint( allowed_models: Final = _health_accessible_model_names(user_api_key_dict, llm_router) restrict_to_allowed_models: Final = allowed_models is not None _llm_model_list: Final = [ - m for m in copy.deepcopy(llm_model_list) if allowed_models is None or m.get("model_name") in allowed_models + m + for m in copy.deepcopy(llm_model_list) + if allowed_models is None + or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id) ] if use_background_health_checks: # The cached background result covers every model. When the diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index b14baf5f8c5..6e5c807afba 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -1,4 +1,5 @@ import asyncio +import copy import json import time from collections.abc import Iterator, Mapping, Sequence @@ -19,6 +20,7 @@ import litellm.proxy.health_endpoints._health_endpoints as _health_endpoints_mod from litellm.litellm_core_utils.health_check_helpers import TEST_IMAGE_BASE64 from litellm.proxy._types import LitellmUserRoles, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.router import Router from litellm.proxy.health_endpoints._health_endpoints import ( _db_health_readiness_check, _show_no_redis_warning, @@ -1652,18 +1654,8 @@ async def test_health_endpoint_resolves_all_team_models_to_team_allowlist(): assert returned_names == {"model-b"}, f"all-team-models key should health-check the team's models: {returned_names}" -class _AccessGroupRouter: - """Router stand-in exposing only the two lookups /health uses to expand a key's model grants.""" - - def __init__(self, access_groups: dict[str, list[str]], model_names: list[str]) -> None: - self._access_groups = access_groups - self._model_names = model_names - - def get_model_access_groups(self, model_name=None, model_access_group=None, team_id=None): - return self._access_groups - - def get_model_names(self, team_id=None): - return self._model_names +def _router_for(model_list: Sequence[Mapping[str, object]]) -> Router: + return Router(model_list=copy.deepcopy(list(model_list))) _ACCESS_GROUP_MODEL_LIST = [ @@ -1678,7 +1670,29 @@ _ACCESS_GROUP_MODEL_LIST = [ "model_info": {"id": "id-openai"}, }, ] -_ACCESS_GROUP_ROUTER = _AccessGroupRouter({"bedrock-group": ["bedrock-nova"]}, ["bedrock-nova", "gpt-5.4-mini"]) +_ACCESS_GROUP_ROUTER = _router_for(_ACCESS_GROUP_MODEL_LIST) +_TEAM_MODEL_LIST = [ + _ACCESS_GROUP_MODEL_LIST[0], + { + "model_name": "bedrock-nova_team-b_9f2c", + "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, + "model_info": { + "id": "id-team-b", + "team_id": "team-b", + "team_public_model_name": "bedrock-nova", + "access_groups": ["bedrock-group"], + }, + }, +] +_TEAM_CACHED_RESULTS = { + "healthy_endpoints": [ + {"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-bedrock"}, + {"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-team-b"}, + ], + "unhealthy_endpoints": [], + "healthy_count": 2, + "unhealthy_count": 0, +} _ACCESS_GROUP_CACHED_RESULTS = { "healthy_endpoints": [ {"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-bedrock"}, @@ -2046,7 +2060,7 @@ async def test_health_endpoint_admin_sees_routing_fields_non_admin_does_not(): # withheld so clients that previously parsed them can detect the change. assert ( non_admin_response.headers.get("Litellm-Health-Field-Notice") - == "api_base and api_version are admin-only on this endpoint" + == "api_base, api_version, aws_bedrock_runtime_endpoint are admin-only on this endpoint" ) assert "Litellm-Health-Field-Notice" not in admin_response.headers @@ -2776,6 +2790,123 @@ def test_clean_endpoint_data_never_displays_credential_fields(credential_field, assert canary not in str(cleaned) +async def _live_probed_model_ids( + model_list: Sequence[Mapping[str, object]], user_api_key_dict: UserAPIKeyAuth +) -> set[str]: + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + captured: dict = {} + + async def fake_perform(**kwargs): + captured["model_list"] = kwargs["model_list"] + return {"healthy_endpoints": [], "unhealthy_endpoints": [], "healthy_count": 0, "unhealthy_count": 0} + + with ( + _proxy_health_globals(model_list, _router_for(model_list)), + patch( # test-quality-ok: the model list handed to the probe is the assertion; no injection seam + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", + side_effect=fake_perform, + ), + ): + await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict) + + return {m["model_info"]["id"] for m in captured["model_list"]} + + +@pytest.mark.asyncio +async def test_health_endpoint_hides_another_teams_deployment_behind_a_shared_access_group(): + """ + Expanding an access group must not reach past the team boundary: a + team-a key holding the group name may not probe team-b's deployment even + though that deployment sits in the same group. + """ + probed = await _live_probed_model_ids( + _TEAM_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"], team_id="team-a"), + ) + + assert probed == {"id-bedrock"} + + +@pytest.mark.asyncio +async def test_health_endpoint_shows_a_teams_own_deployment_by_its_public_name(): + """ + A team key names its team deployment by ``team_public_model_name``, while + the proxy model list carries the internal ``__`` + name; the deployment must still be probed for its own team. + """ + probed = await _live_probed_model_ids( + _TEAM_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"], team_id="team-b"), + ) + + assert probed == {"id-bedrock", "id-team-b"} + + +@pytest.mark.asyncio +async def test_health_endpoint_hides_another_teams_deployment_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"], team_id="team-a"), + model=None, + model_id=None, + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-bedrock"] + assert result["healthy_count"] == 1 + + +def test_health_test_connection_keeps_error_and_raw_request_through_the_allowlist(monkeypatch): + """ + The dashboard's Test Connect button reads ``result.error`` and + ``result.raw_request_typed_dict`` from /health/test_connection, so the + allowlist must keep both while dropping the probe's own params. + """ + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + app = FastAPI() + app.include_router(_health_endpoints_module.router) + app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + client = TestClient(app) + + with ( + patch( # test-quality-ok: the endpoint reads the proxy-global DB client and 500s when it is None; it has no injection seam + "litellm.proxy.proxy_server.prisma_client", MagicMock() + ), + respx.mock(assert_all_called=True) as respx_mock, + ): + respx_mock.post(host="api.openai.com", path="/v1/chat/completions").respond( + status_code=401, json={"error": {"message": "Incorrect API key provided"}} + ) + response = client.post( + "/health/test_connection", + json={ + "mode": "chat", + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "sk-test", "timeout": 7}, + }, + ) + + assert response.status_code == 200, response.text + body = response.json() + assert body["status"] == "error" + assert "Incorrect API key provided" in body["result"]["error"] + assert "api.openai.com" in body["result"]["raw_request_typed_dict"]["raw_request_api_base"] + assert not {"api_key", "timeout", "exception"} & set(body["result"]) + + def test_clean_endpoint_data_keeps_only_json_safe_diagnostics(): """ LIT-6907: _clean_endpoint_data used to copy every litellm_param not on a @@ -2796,6 +2927,7 @@ def test_clean_endpoint_data_keeps_only_json_safe_diagnostics(): "api_key": "CANARY-API-KEY", "x-ratelimit-remaining-requests": 99, "raw_request_typed_dict": {"raw_request_api_base": "https://example.test"}, + "aws_bedrock_runtime_endpoint": "https://vpce-bedrock.example.test", }, details=True, ) @@ -2806,6 +2938,7 @@ def test_clean_endpoint_data_keeps_only_json_safe_diagnostics(): "aws_region_name": "us-east-1", "x-ratelimit-remaining-requests": 99, "raw_request_typed_dict": {"raw_request_api_base": "https://example.test"}, + "aws_bedrock_runtime_endpoint": "https://vpce-bedrock.example.test", } assert jsonable_encoder(cleaned) == cleaned From 471f51cc4deb5aa2db57bff70c5eac32eaa3bfc3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:29:09 -0700 Subject: [PATCH 3/9] fix(proxy): keep another team's deployment out of /health for keys with no team --- .../health_endpoints/_health_endpoints.py | 4 +- .../health_endpoints/test_health_endpoints.py | 38 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index cc9ad1eb0cb..04388c773a0 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -880,7 +880,9 @@ def _health_accessible_model_names( def _caller_may_probe_deployment( deployment: Mapping[str, object], allowed_models: frozenset[str], llm_router: Router | None, team_id: str | None ) -> bool: - """Same deployment visibility rule as request auth: another team's deployment is never in scope.""" + """Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included.""" + if not Router._deployment_usable_by_team(deployment, team_id): + return False if llm_router is None: return deployment.get("model_name") in allowed_models model: Final = dict(deployment) diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index 6e5c807afba..f4ed50fdb5f 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -2830,6 +2830,21 @@ async def test_health_endpoint_hides_another_teams_deployment_behind_a_shared_ac assert probed == {"id-bedrock"} +@pytest.mark.asyncio +async def test_health_endpoint_hides_team_deployments_from_a_key_with_no_team(): + """ + Routing never serves a team-owned deployment to a caller without a team + (``filter_team_based_models``), so a team-less access-group key must not + probe team-b's deployment with team-b's credentials either. + """ + probed = await _live_probed_model_ids( + _TEAM_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"], team_id=None), + ) + + assert probed == {"id-bedrock"} + + @pytest.mark.asyncio async def test_health_endpoint_shows_a_teams_own_deployment_by_its_public_name(): """ @@ -2868,6 +2883,29 @@ async def test_health_endpoint_hides_another_teams_deployment_on_background_cach assert result["healthy_count"] == 1 +@pytest.mark.asyncio +async def test_health_endpoint_hides_team_deployments_from_a_key_with_no_team_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"], team_id=None), + model=None, + model_id=None, + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-bedrock"] + assert result["healthy_count"] == 1 + + def test_health_test_connection_keeps_error_and_raw_request_through_the_allowlist(monkeypatch): """ The dashboard's Test Connect button reads ``result.error`` and From 69a45e81ccb49c6ee4b874ec21fd70668caff036 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:40:00 -0700 Subject: [PATCH 4/9] fix(proxy): refuse /health targets outside the caller's scope instead of probing the rest --- .../health_endpoints/_health_endpoints.py | 11 ++- .../health_endpoints/test_health_endpoints.py | 94 +++++++++++++++---- 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 04388c773a0..b8c56037206 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -1127,6 +1127,16 @@ async def health_endpoint( if allowed_models is None or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id) ] + targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id) + if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={ + "error": f"key not allowed to health-check model_id {model_id}" + if model_id + else f"key not allowed to health-check model {model}" + }, + ) if use_background_health_checks: # The cached background result covers every model. When the # caller targets a specific model/model_id we have to narrow the @@ -1134,7 +1144,6 @@ async def health_endpoint( # healthy_count, otherwise an unhealthy "foo" combined with any # other healthy model would still report healthy_count > 0 and # the targeted-503 path would never fire. - targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id) if restrict_to_allowed_models: allowed_model_ids: Final = { (m.get("model_info") or {}).get("id") diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index f4ed50fdb5f..c03284b72b6 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -1517,7 +1517,7 @@ async def test_health_endpoint_filters_model_list_by_user_access(): ): from fastapi import Response - await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict) + await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict, model=None, model_id=None) assert "model_list" in captured, "health_endpoint did not call _perform_health_check_and_save" returned_names = {m["model_name"] for m in captured["model_list"]} @@ -1580,7 +1580,7 @@ async def test_health_endpoint_keeps_full_model_list_for_all_proxy_models(): ): from fastapi import Response - await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict) + await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict, model=None, model_id=None) returned_names = {m["model_name"] for m in captured["model_list"]} assert returned_names == { @@ -1648,7 +1648,7 @@ async def test_health_endpoint_resolves_all_team_models_to_team_allowlist(): ): from fastapi import Response - await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict) + await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict, model=None, model_id=None) returned_names = {m["model_name"] for m in captured["model_list"]} assert returned_names == {"model-b"}, f"all-team-models key should health-check the team's models: {returned_names}" @@ -1769,6 +1769,8 @@ async def test_health_endpoint_expands_access_group_on_live_path(): await health_endpoint( response=Response(), user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"]), + model=None, + model_id=None, ) assert [m["model_name"] for m in captured["model_list"]] == ["bedrock-nova"] @@ -2149,7 +2151,7 @@ async def test_health_endpoint_blocks_cross_scope_model_id_under_background_cach cache filter was driven by an unvalidated ID and the global cache leaked id-b's entry to the caller. """ - from fastapi import Response + from fastapi import HTTPException, Response from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.health_endpoints._health_endpoints import health_endpoint @@ -2200,21 +2202,18 @@ async def test_health_endpoint_blocks_cross_scope_model_id_under_background_cach ): # Calling with model="model-b" rather than model_id="id-b" because # the model_id branch raises 404 when llm_router is None. The bug - # being verified is the same: targeted resolver must drop entries - # not in the caller's scoped model_list. With the fix, the result - # has no leaked endpoints and the targeted-503 path fires. - result = await health_endpoint( - response=response, - user_api_key_dict=user_api_key_dict, - model="model-b", - model_id=None, - ) + # being verified is the same: a target outside the caller's scoped + # model_list is refused before the cache is read. + with pytest.raises(HTTPException) as refused: + await health_endpoint( + response=response, + user_api_key_dict=user_api_key_dict, + model="model-b", + model_id=None, + ) - leaked_ids = {ep.get("model_id") for ep in result.get("healthy_endpoints", [])} - leaked_ids |= {ep.get("model_id") for ep in result.get("unhealthy_endpoints", [])} - assert "id-b" not in leaked_ids, "background cache leaked an out-of-scope deployment to a scoped caller" - assert result["healthy_count"] == 0 - assert response.status_code == 503 + assert refused.value.status_code == 403 + assert "leaky-internal.test" not in str(refused.value.detail) @pytest.mark.asyncio @@ -2810,7 +2809,7 @@ async def _live_probed_model_ids( side_effect=fake_perform, ), ): - await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict) + await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict, model=None, model_id=None) return {m["model_info"]["id"] for m in captured["model_list"]} @@ -2883,6 +2882,63 @@ async def test_health_endpoint_hides_another_teams_deployment_on_background_cach assert result["healthy_count"] == 1 +@pytest.mark.asyncio +async def test_health_endpoint_refuses_a_targeted_deployment_outside_the_callers_scope_on_live_path(): + """ + A scoped key asking for a deployment it may not see must get a 403 and no + probe at all: probing the rest of its scope instead would report another + deployment's health under the requested id and store it as such. + """ + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + fake_perform = AsyncMock() + + with ( + _proxy_health_globals(_TEAM_MODEL_LIST, _router_for(_TEAM_MODEL_LIST)), + patch( # test-quality-ok: the probe must never run; no injection seam + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", + fake_perform, + ), + pytest.raises(HTTPException) as excinfo, + ): + await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"], team_id=None), + model=None, + model_id="id-team-b", + ) + + assert excinfo.value.status_code == 403 + fake_perform.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_health_endpoint_refuses_a_targeted_deployment_outside_the_callers_scope_on_background_cache_path(): + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ), + pytest.raises(HTTPException) as excinfo, + ): + await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-group"], team_id="team-a"), + model="bedrock-nova_team-b_9f2c", + model_id=None, + ) + + assert excinfo.value.status_code == 403 + + @pytest.mark.asyncio async def test_health_endpoint_hides_team_deployments_from_a_key_with_no_team_on_background_cache_path(): from fastapi import Response From 8b3faa6ed83290828ae3520aa2d35b11f9066fe6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 09:53:53 -0700 Subject: [PATCH 5/9] fix(proxy): keep /health inside the caller's team for unrestricted non-admin keys --- .../health_endpoints/_health_endpoints.py | 16 ++++-- .../health_endpoints/test_health_endpoints.py | 53 +++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index b8c56037206..bf0200acfa9 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -878,11 +878,17 @@ def _health_accessible_model_names( def _caller_may_probe_deployment( - deployment: Mapping[str, object], allowed_models: frozenset[str], llm_router: Router | None, team_id: str | None + deployment: Mapping[str, object], + allowed_models: frozenset[str] | None, + llm_router: Router | None, + team_id: str | None, + caller_is_admin: bool, ) -> bool: """Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included.""" - if not Router._deployment_usable_by_team(deployment, team_id): + if not caller_is_admin and not Router._deployment_usable_by_team(deployment, team_id): return False + if allowed_models is None: + return True if llm_router is None: return deployment.get("model_name") in allowed_models model: Final = dict(deployment) @@ -1120,12 +1126,12 @@ async def health_endpoint( detail={"error": "Model list not initialized"}, ) allowed_models: Final = _health_accessible_model_names(user_api_key_dict, llm_router) - restrict_to_allowed_models: Final = allowed_models is not None + restrict_to_allowed_models: Final = not is_admin or allowed_models is not None _llm_model_list: Final = [ m for m in copy.deepcopy(llm_model_list) - if allowed_models is None - or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id) + if not restrict_to_allowed_models + or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id, is_admin) ] targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id) if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids: diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index c03284b72b6..bbcae2fb68f 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -1836,6 +1836,8 @@ async def test_health_endpoint_treats_no_team_all_team_models_as_unrestricted(): user_api_key_dict=UserAPIKeyAuth( api_key="hashed-test-key", models=[SpecialModelNames.all_team_models.value], team_id=None ), + model=None, + model_id=None, ) assert {m["model_name"] for m in captured["model_list"]} == {"bedrock-nova", "gpt-5.4-mini"} @@ -2844,6 +2846,57 @@ async def test_health_endpoint_hides_team_deployments_from_a_key_with_no_team(): assert probed == {"id-bedrock"} +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("team_id", "expected_ids"), + [(None, {"id-bedrock"}), ("team-a", {"id-bedrock"}), ("team-b", {"id-bedrock", "id-team-b"})], +) +async def test_health_endpoint_keeps_an_unrestricted_non_admin_key_to_its_own_team(team_id, expected_ids): + """ + A key with no model restriction is still bound by routing's team rule: + it may probe global deployments and its own team's, never another team's. + """ + probed = await _live_probed_model_ids( + _TEAM_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=[], team_id=team_id), + ) + + assert probed == expected_ids + + +@pytest.mark.asyncio +async def test_health_endpoint_lets_a_proxy_admin_probe_every_teams_deployment(): + probed = await _live_probed_model_ids( + _TEAM_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=[], user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert probed == {"id-bedrock", "id-team-b"} + + +@pytest.mark.asyncio +async def test_health_endpoint_keeps_an_unrestricted_non_admin_key_to_its_own_team_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=[], team_id="team-a"), + model=None, + model_id=None, + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-bedrock"] + assert result["healthy_count"] == 1 + + @pytest.mark.asyncio async def test_health_endpoint_shows_a_teams_own_deployment_by_its_public_name(): """ From 787f2dee0c3d9e895c14adfccd21ca7e090fea24 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:52:59 -0700 Subject: [PATCH 6/9] fix(health): match team public model names when targeting /health by model --- litellm/proxy/health_check.py | 9 ++- .../health_endpoints/_health_endpoints.py | 9 +-- .../health_endpoints/test_health_endpoints.py | 67 ++++++++++++++++++- .../proxy/test_health_check_functions.py | 29 ++++++++ 4 files changed, 107 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 78bb01aa6ac..8cda318b937 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -258,6 +258,13 @@ def _deployment_model(deployment: Mapping[str, object]) -> str | None: return params.get("model") if isinstance(params, Mapping) else None +def deployment_answers_to(deployment: Mapping[str, object], model_name: str) -> bool: + """True when `model_name` is the deployment's model_name or the public name a team key reaches it by.""" + info: Final = deployment.get("model_info") + public_name: Final = info.get("team_public_model_name") if isinstance(info, Mapping) else None + return model_name in (deployment.get("model_name"), public_name) + + def _narrow_to_target( model_list: Sequence[Mapping[str, object]], model: str | None, model_id: str | None ) -> tuple[Mapping[str, object], ...]: @@ -268,7 +275,7 @@ def _narrow_to_target( if model is None: return tuple(model_list) by_param: Final = tuple(x for x in model_list if _deployment_model(x) == model) - return by_param or tuple(x for x in model_list if x.get("model_name") == model) + return by_param or tuple(x for x in model_list if deployment_answers_to(x, model)) def _is_strategy_router_deployment(litellm_params: Mapping[str, object]) -> bool: diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index c66eafc724f..267b094526b 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -50,6 +50,7 @@ from litellm.proxy.health_check import ( ADMIN_ONLY_HEALTH_DISPLAY_PARAMS, _clean_endpoint_data, _update_litellm_params_for_health_check, + deployment_answers_to, health_check_filter_kwargs_from_general_settings, perform_health_check, run_with_timeout, @@ -930,9 +931,9 @@ def _resolve_targeted_model_ids(model_list: list, model: str | None, model_id: s deployment IDs the response should be scoped to. Mirrors the live-path semantics in ``perform_health_check()``: ``model`` - matches either the deployment's ``model_name`` alias or its - ``litellm_params.model`` provider string. ``model_id`` matches - ``model_info.id``. + matches the deployment's ``model_name`` alias, its ``litellm_params.model`` + provider string, or the ``model_info.team_public_model_name`` a team key + reaches it by. ``model_id`` matches ``model_info.id``. Both query params are validated against the supplied ``model_list``. Callers pass an already-scoped list (filtered to the caller's allowed @@ -956,7 +957,7 @@ def _resolve_targeted_model_ids(model_list: list, model: str | None, model_id: s continue if model: litellm_model = (m.get("litellm_params") or {}).get("model") - if m.get("model_name") == model or litellm_model == model: + if litellm_model == model or deployment_answers_to(m, model): target_ids.add(deployment_id) return target_ids diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index 022cf1e5505..e0cf6d97141 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -2802,7 +2802,7 @@ def test_clean_endpoint_data_never_displays_credential_fields(credential_field, async def _live_probed_model_ids( - model_list: Sequence[Mapping[str, object]], user_api_key_dict: UserAPIKeyAuth + model_list: Sequence[Mapping[str, object]], user_api_key_dict: UserAPIKeyAuth, model: str | None = None ) -> set[str]: from fastapi import Response @@ -2821,7 +2821,7 @@ async def _live_probed_model_ids( side_effect=fake_perform, ), ): - await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict, model=None, model_id=None) + await health_endpoint(response=Response(), user_api_key_dict=user_api_key_dict, model=model, model_id=None) return {m["model_info"]["id"] for m in captured["model_list"]} @@ -3025,6 +3025,69 @@ async def test_health_endpoint_hides_team_deployments_from_a_key_with_no_team_on assert result["healthy_count"] == 1 +_TEAM_ONLY_MODEL_LIST = [_TEAM_MODEL_LIST[1]] + + +@pytest.mark.asyncio +async def test_health_endpoint_probes_a_team_only_deployment_by_its_public_name_on_live_path(): + """ + A team key targets its deployment by ``team_public_model_name``; when that + name resolves to nothing but the team deployment, the probe must run rather + than 403 as if the key were out of scope. + """ + probed = await _live_probed_model_ids( + _TEAM_ONLY_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"], team_id="team-b"), + model="bedrock-nova", + ) + + assert probed == {"id-team-b"} + + +@pytest.mark.asyncio +async def test_health_endpoint_returns_a_team_only_deployment_by_its_public_name_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_ONLY_MODEL_LIST, + _router_for(_TEAM_ONLY_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"], team_id="team-b"), + model="bedrock-nova", + model_id=None, + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-team-b"] + + +@pytest.mark.asyncio +async def test_health_endpoint_targets_both_deployments_behind_a_shared_public_name_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"], team_id="team-b"), + model="bedrock-nova", + model_id=None, + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-bedrock", "id-team-b"] + + def test_health_test_connection_keeps_error_and_raw_request_through_the_allowlist(monkeypatch): """ The dashboard's Test Connect button reads ``result.error`` and diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index fdae11d517a..137eda223d1 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -623,6 +623,35 @@ async def test_perform_health_check_and_save_forwards_skip_disabled_background_f assert call_kwargs["health_check_skip_disabled_background_models"] is True +@pytest.mark.asyncio +async def test_perform_health_check_narrows_to_a_team_deployment_by_its_public_name(): + """``/health?model=`` must probe the team deployment, not an empty list.""" + from litellm.proxy.health_check import perform_health_check + + team_deployment = { + "model_name": "bedrock-nova_team-b_9f2c", + "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, + "model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"}, + } + other_deployment = { + "model_name": "gpt-5.4-mini", + "litellm_params": {"model": "openai/gpt-5.4-mini"}, + "model_info": {"id": "id-openai"}, + } + probe = AsyncMock(return_value=([{"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-team-b"}], [], {})) + + with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam + "litellm.proxy.health_check._perform_health_check", probe + ): + healthy, unhealthy, _ = await perform_health_check( + model_list=[team_deployment, other_deployment], model="bedrock-nova" + ) + + assert [m["model_info"]["id"] for m in probe.call_args.args[0]] == ["id-team-b"] + assert [ep["model_id"] for ep in healthy] == ["id-team-b"] + assert unhealthy == [] + + def test_parse_background_health_check_model_groups_unset_returns_none(): from litellm.proxy.health_check import parse_background_health_check_model_groups From 6209f0694bfd7285deea4095faea4e850a7fe007 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 11 Sep 2026 19:49:37 -0700 Subject: [PATCH 7/9] fix(health): keep team public names to the owning team and let model_id win over model A public name a team publishes its own deployment copy under now targets that copy only for a caller from that team, so an admin or another team probing the shared name gets the global deployment alone model_id wins when paired with model: a foreign id still gets the 403, and an id no deployment carries gets the 404 of the lone-id path, before any probe runs or a result is stored under it cache_health_check_results accepts the Mapping sequences perform_health_check returns --- litellm/proxy/health_check.py | 19 ++- .../shared_health_check_manager.py | 5 +- .../health_endpoints/_health_endpoints.py | 61 +++---- .../health_endpoints/test_health_endpoints.py | 159 ++++++++++++++++++ .../proxy/test_health_check_functions.py | 27 ++- .../proxy/test_health_check_max_tokens.py | 6 +- 6 files changed, 231 insertions(+), 46 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 8cda318b937..9ca31c8b626 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -258,15 +258,18 @@ def _deployment_model(deployment: Mapping[str, object]) -> str | None: return params.get("model") if isinstance(params, Mapping) else None -def deployment_answers_to(deployment: Mapping[str, object], model_name: str) -> bool: - """True when `model_name` is the deployment's model_name or the public name a team key reaches it by.""" +def deployment_answers_to(deployment: Mapping[str, object], model_name: str, team_id: str | None) -> bool: + """True when `model_name` is the deployment's model_name or the public name the caller's own team reaches it by.""" info: Final = deployment.get("model_info") public_name: Final = info.get("team_public_model_name") if isinstance(info, Mapping) else None - return model_name in (deployment.get("model_name"), public_name) + owner_team_id: Final = info.get("team_id") if isinstance(info, Mapping) else None + return model_name == deployment.get("model_name") or ( + model_name == public_name and team_id is not None and owner_team_id == team_id + ) def _narrow_to_target( - model_list: Sequence[Mapping[str, object]], model: str | None, model_id: str | None + model_list: Sequence[Mapping[str, object]], model: str | None, model_id: str | None, team_id: str | None ) -> tuple[Mapping[str, object], ...]: """Narrow to the requested deployment. An id matching nothing keeps the whole list.""" if model_id is not None: @@ -275,7 +278,7 @@ def _narrow_to_target( if model is None: return tuple(model_list) by_param: Final = tuple(x for x in model_list if _deployment_model(x) == model) - return by_param or tuple(x for x in model_list if deployment_answers_to(x, model)) + return by_param or tuple(x for x in model_list if deployment_answers_to(x, model, team_id)) def _is_strategy_router_deployment(litellm_params: Mapping[str, object]) -> bool: @@ -820,13 +823,15 @@ async def perform_health_check( instrumentation_context: dict | None = None, health_check_skip_disabled_background_models: bool = False, router: "Router | None" = None, + team_id: str | None = None, ): """ Perform a health check on the system. When model_id is provided, only the deployment with that id is checked (so models that share the same name but have different ids are checked separately). - When model (name) is provided, all deployments matching that name are checked. + When model (name) is provided, all deployments matching that name are checked; + a team's public model name only matches for a caller from that team (``team_id``). When ``health_check_skip_disabled_background_models`` is True (via ``general_settings.health_check_skip_disabled_background_models``), deployments @@ -857,7 +862,7 @@ async def perform_health_check( cycle_start_time: Final = time.monotonic() requested_model_count: Final = len(model_list) skip_disabled: Final = health_check_skip_disabled_background_models - narrowed: Final = _health_check_eligible(_narrow_to_target(model_list, model, model_id), skip_disabled) + narrowed: Final = _health_check_eligible(_narrow_to_target(model_list, model, model_id, team_id), skip_disabled) if not narrowed: if instrumentation_enabled: logger.debug( diff --git a/litellm/proxy/health_check_utils/shared_health_check_manager.py b/litellm/proxy/health_check_utils/shared_health_check_manager.py index f12cee4b636..7dbbd55d87f 100644 --- a/litellm/proxy/health_check_utils/shared_health_check_manager.py +++ b/litellm/proxy/health_check_utils/shared_health_check_manager.py @@ -1,6 +1,7 @@ import asyncio import json import time +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Any, Final from litellm._logging import verbose_proxy_logger @@ -143,8 +144,8 @@ class SharedHealthCheckManager: async def cache_health_check_results( self, - healthy_endpoints: list[dict[str, Any]], - unhealthy_endpoints: list[dict[str, Any]], + healthy_endpoints: Sequence[Mapping[str, object]], + unhealthy_endpoints: Sequence[Mapping[str, object]], ) -> None: """ Cache health check results in Redis. diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 267b094526b..dcef0fcd58a 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -925,41 +925,31 @@ def _caller_may_probe_deployment( ) -def _resolve_targeted_model_ids(model_list: list, model: str | None, model_id: str | None) -> set | None: +def _resolve_targeted_model_ids( + model_list: list, model: str | None, model_id: str | None, team_id: str | None +) -> set | None: """ Resolve a ``/health`` ``model`` / ``model_id`` query param to the set of - deployment IDs the response should be scoped to. + deployment IDs the response should be scoped to, mirroring the live-path + narrowing in ``perform_health_check()``: ``model_id`` wins when given and + matches ``model_info.id`` only; ``model`` matches the deployment's + ``model_name`` alias, its ``litellm_params.model`` provider string, or the + ``model_info.team_public_model_name`` the caller's own team reaches it by. - Mirrors the live-path semantics in ``perform_health_check()``: ``model`` - matches the deployment's ``model_name`` alias, its ``litellm_params.model`` - provider string, or the ``model_info.team_public_model_name`` a team key - reaches it by. ``model_id`` matches ``model_info.id``. - - Both query params are validated against the supplied ``model_list``. - Callers pass an already-scoped list (filtered to the caller's allowed - models for non-admins, full list for admins), so a ``model_id`` that - isn't present resolves to an empty set rather than a single-element - set — preventing a non-admin from reading another deployment's cached - health entry by guessing its ID. - - Returns ``None`` when no targeting is requested — callers should treat - that as "no filter." + Callers pass an already-scoped list, so a ``model_id`` outside the + caller's scope resolves to an empty set and never to the unvalidated id. + Returns ``None`` when no targeting is requested. """ - if not model and not model_id: + if model_id: + return {i for m in model_list if (i := (m.get("model_info") or {}).get("id")) == model_id} + if not model: return None - target_ids: Final[set] = set() - for m in model_list: - deployment_id = (m.get("model_info") or {}).get("id") - if not deployment_id: - continue - if model_id and deployment_id == model_id: - target_ids.add(deployment_id) - continue - if model: - litellm_model = (m.get("litellm_params") or {}).get("model") - if litellm_model == model or deployment_answers_to(m, model): - target_ids.add(deployment_id) - return target_ids + return { + i + for m in model_list + if (i := (m.get("model_info") or {}).get("id")) + and ((m.get("litellm_params") or {}).get("model") == model or deployment_answers_to(m, model, team_id)) + } def _filter_health_check_results_by_model_ids(results: dict, allowed_model_ids: set) -> dict: @@ -1040,8 +1030,12 @@ def _health_endpoint_resolve_target_model_name( model_id: str | None, llm_router, ) -> str | None: - """Map ``model_id`` (without ``model``) to ``model_name`` for live health checks.""" - if not model_id or model: + """Map ``model_id`` to its deployment's ``model_name`` for live health checks. + + ``model_id`` wins over ``model``, so an id no deployment carries is a 404 even + when it is paired with a known name. + """ + if not model_id: return model if llm_router is None: raise HTTPException( @@ -1161,7 +1155,7 @@ async def health_endpoint( if not restrict_to_allowed_models or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id, is_admin) ] - targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id) + targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id) if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, @@ -1228,6 +1222,7 @@ async def health_endpoint( model_id=model_id, max_concurrency=health_check_concurrency, router=llm_router, + team_id=user_api_key_dict.team_id, **_hc_filter, ) return _post_process(router_result) diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index e0cf6d97141..c5950f6f30e 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -2357,6 +2357,7 @@ async def test_health_endpoint_returns_503_when_requested_model_has_no_healthy_e response=response, user_api_key_dict=user_api_key_dict, model="model-a", + model_id=None, ) assert response.status_code == 503 @@ -2417,6 +2418,7 @@ async def test_health_endpoint_returns_200_when_requested_model_has_healthy_endp response=response, user_api_key_dict=user_api_key_dict, model="model-a", + model_id=None, ) assert response.status_code == 200 @@ -3088,6 +3090,161 @@ async def test_health_endpoint_targets_both_deployments_behind_a_shared_public_n assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-bedrock", "id-team-b"] +async def _live_narrowed_model_ids( + model_list: Sequence[Mapping[str, object]], + user_api_key_dict: UserAPIKeyAuth, + model: str | None = None, + model_id: str | None = None, +) -> set[str]: + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + async def fake_probe(model_list, details=True, max_concurrency=None, instrumentation_context=None): + probed = [{"model": m["litellm_params"]["model"], "model_id": m["model_info"]["id"]} for m in model_list] + return probed, [], {} + + with ( + _proxy_health_globals(model_list, _router_for(model_list)), + patch( # test-quality-ok: the probe is the provider edge; which deployments reach it is the assertion + "litellm.proxy.health_check._perform_health_check", side_effect=fake_probe + ), + ): + result = await health_endpoint( + response=Response(), user_api_key_dict=user_api_key_dict, model=model, model_id=model_id + ) + + return {ep["model_id"] for ep in result["healthy_endpoints"]} + + +_ADMIN_OUTSIDE_TEAM_B = UserAPIKeyAuth(api_key="hashed-test-key", models=[], user_role=LitellmUserRoles.PROXY_ADMIN) + + +@pytest.mark.asyncio +async def test_health_endpoint_keeps_an_admin_probe_by_name_off_other_teams_public_copies(): + """ + An admin outside team-b asks for ``bedrock-nova``. Team-b's copy answers to + that name only for team-b (routing keys public names by team), so probing + it too would spend team-b's credentials and let a healthy team copy mask a + down global deployment as 200. + """ + probed = await _live_narrowed_model_ids(_TEAM_MODEL_LIST, _ADMIN_OUTSIDE_TEAM_B, model="bedrock-nova") + + assert probed == {"id-bedrock"} + + +@pytest.mark.asyncio +async def test_health_endpoint_probes_both_deployments_behind_a_shared_public_name_for_the_owning_team(): + probed = await _live_narrowed_model_ids( + _TEAM_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"], team_id="team-b"), + model="bedrock-nova", + ) + + assert probed == {"id-bedrock", "id-team-b"} + + +@pytest.mark.asyncio +async def test_health_endpoint_keeps_an_admin_probe_by_name_off_other_teams_public_copies_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), user_api_key_dict=_ADMIN_OUTSIDE_TEAM_B, model="bedrock-nova", model_id=None + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-bedrock"] + + +def test_resolve_targeted_model_ids_lets_model_id_win_over_model(): + resolve = _health_endpoints_module._resolve_targeted_model_ids + + assert resolve(_TEAM_MODEL_LIST, "bedrock-nova", "id-team-b", None) == {"id-team-b"} + assert resolve([_TEAM_MODEL_LIST[0]], "bedrock-nova", "id-team-b", None) == set() + assert resolve(_TEAM_MODEL_LIST, "bedrock-nova", None, None) == {"id-bedrock"} + assert resolve(_TEAM_MODEL_LIST, "bedrock-nova", None, "team-b") == {"id-bedrock", "id-team-b"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [False, True]) +async def test_health_endpoint_rejects_an_in_scope_model_paired_with_a_foreign_model_id(use_background_health_checks): + """ + A key scoped to ``bedrock-nova`` pairs that name with another team's + deployment id. The in-scope name must not carry the foreign id past the + 403: the live path narrows by id first, so the caller's own deployment + would be probed and its result stored under the foreign id. + """ + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=use_background_health_checks, + health_check_results=_TEAM_CACHED_RESULTS, + ), + patch( # test-quality-ok: the probe must never run; the endpoint has no injection seam for it + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", new_callable=AsyncMock + ) as probe, + pytest.raises(HTTPException) as refused, + ): + await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"]), + model="bedrock-nova", + model_id="id-team-b", + ) + + assert refused.value.status_code == 403 + assert "id-team-b" in str(refused.value.detail) + probe.assert_not_awaited() + + +@pytest.mark.parametrize("use_background_health_checks", [False, True]) +@pytest.mark.asyncio +async def test_health_endpoint_returns_404_for_a_model_paired_with_an_unknown_model_id(use_background_health_checks): + """ + ``model_id`` wins over ``model``: pairing a known name with an id no + deployment carries gets the same 404 as the lone unknown id, before any + probe runs or a result is stored under the unknown id. + """ + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals( + _TEAM_MODEL_LIST, + _router_for(_TEAM_MODEL_LIST), + use_background_health_checks=use_background_health_checks, + health_check_results=_TEAM_CACHED_RESULTS, + ), + patch( # test-quality-ok: the probe must never run; the endpoint has no injection seam for it + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", new_callable=AsyncMock + ) as probe, + pytest.raises(HTTPException) as refused, + ): + await health_endpoint( + response=Response(), + user_api_key_dict=_ADMIN_OUTSIDE_TEAM_B, + model="bedrock-nova", + model_id="id-nobody-has", + ) + + assert refused.value.status_code == 404 + assert "id-nobody-has" in str(refused.value.detail) + probe.assert_not_awaited() + + def test_health_test_connection_keeps_error_and_raw_request_through_the_allowlist(monkeypatch): """ The dashboard's Test Connect button reads ``result.error`` and @@ -3198,6 +3355,8 @@ async def test_health_endpoint_result_survives_non_json_safe_deployment_params() result = await health_endpoint( response=Response(), user_api_key_dict=UserAPIKeyAuth(api_key="hashed-admin-key", user_role=LitellmUserRoles.PROXY_ADMIN), + model=None, + model_id=None, ) encoded = jsonable_encoder(result) diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index 137eda223d1..af20b80a718 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -644,7 +644,7 @@ async def test_perform_health_check_narrows_to_a_team_deployment_by_its_public_n "litellm.proxy.health_check._perform_health_check", probe ): healthy, unhealthy, _ = await perform_health_check( - model_list=[team_deployment, other_deployment], model="bedrock-nova" + model_list=[team_deployment, other_deployment], model="bedrock-nova", team_id="team-b" ) assert [m["model_info"]["id"] for m in probe.call_args.args[0]] == ["id-team-b"] @@ -652,6 +652,31 @@ async def test_perform_health_check_narrows_to_a_team_deployment_by_its_public_n assert unhealthy == [] +@pytest.mark.asyncio +@pytest.mark.parametrize("team_id", [None, "team-a"]) +async def test_perform_health_check_keeps_a_public_name_to_the_owning_team(team_id): + """A team's public model name is not a global alias: callers outside the team must not probe its deployment.""" + from litellm.proxy.health_check import perform_health_check + + team_deployment = { + "model_name": "bedrock-nova_team-b_9f2c", + "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, + "model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"}, + } + probe = AsyncMock(return_value=([{"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-team-b"}], [], {})) + + with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam + "litellm.proxy.health_check._perform_health_check", probe + ): + healthy, unhealthy, _ = await perform_health_check( + model_list=[team_deployment], model="bedrock-nova", team_id=team_id + ) + + probe.assert_not_awaited() + assert healthy == [] + assert unhealthy == [] + + def test_parse_background_health_check_model_groups_unset_returns_none(): from litellm.proxy.health_check import parse_background_health_check_model_groups diff --git a/tests/test_litellm/proxy/test_health_check_max_tokens.py b/tests/test_litellm/proxy/test_health_check_max_tokens.py index 97e308d7c3c..dd3669644af 100644 --- a/tests/test_litellm/proxy/test_health_check_max_tokens.py +++ b/tests/test_litellm/proxy/test_health_check_max_tokens.py @@ -872,9 +872,9 @@ def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list(): """Pinned because the disabled-dependency fix moved this filter into its own helper.""" deployments = [{"model_name": "a", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a-1"}}] - assert hc_module._narrow_to_target(deployments, None, "no-such-id") == tuple(deployments) - assert hc_module._narrow_to_target(deployments, None, "a-1") == tuple(deployments) - assert hc_module._narrow_to_target(deployments, "a", None) == tuple(deployments) + assert hc_module._narrow_to_target(deployments, None, "no-such-id", None) == tuple(deployments) + assert hc_module._narrow_to_target(deployments, None, "a-1", None) == tuple(deployments) + assert hc_module._narrow_to_target(deployments, "a", None, None) == tuple(deployments) def _nested_router_fixture(parent_tier: str): From eaf3d8ad3ec03cdb27f4305236e8f64cd1357f5b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 11 Sep 2026 20:12:39 -0700 Subject: [PATCH 8/9] fix(health): target a model name the way a request for it routes A team's copies published under the name win, then deployments named that way, then a public name only another team's deployment carries (an admin reaches it, routing does too). The endpoint resolver and the live narrowing share one rule. --- litellm/proxy/health_check.py | 54 ++++++++++--- .../health_endpoints/_health_endpoints.py | 11 ++- .../health_endpoints/test_health_endpoints.py | 78 +++++++++++++++++-- .../proxy/test_health_check_functions.py | 51 +++++++++++- 4 files changed, 169 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 9ca31c8b626..d774fd6e191 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -258,14 +258,47 @@ def _deployment_model(deployment: Mapping[str, object]) -> str | None: return params.get("model") if isinstance(params, Mapping) else None -def deployment_answers_to(deployment: Mapping[str, object], model_name: str, team_id: str | None) -> bool: - """True when `model_name` is the deployment's model_name or the public name the caller's own team reaches it by.""" +def _owner_team_id(deployment: Mapping[str, object]) -> str | None: info: Final = deployment.get("model_info") - public_name: Final = info.get("team_public_model_name") if isinstance(info, Mapping) else None - owner_team_id: Final = info.get("team_id") if isinstance(info, Mapping) else None - return model_name == deployment.get("model_name") or ( - model_name == public_name and team_id is not None and owner_team_id == team_id + owner: Final = info.get("team_id") if isinstance(info, Mapping) else None + return owner if isinstance(owner, str) else None + + +def _team_public_model_name(deployment: Mapping[str, object]) -> str | None: + info: Final = deployment.get("model_info") + name: Final = info.get("team_public_model_name") if isinstance(info, Mapping) else None + return name if isinstance(name, str) else None + + +def _deployments_routed_by_name( + model_list: Sequence[Mapping[str, object]], model_name: str, team_id: str | None +) -> tuple[Mapping[str, object], ...]: + """The deployments a request for ``model_name`` from this caller routes to. + + A team's own copies published under that name win, then deployments carrying it as + ``model_name``. A caller with no team reaches a public name only when nothing carries + it as ``model_name``, and only an admin still has another team's deployment in a + scoped ``model_list`` by then. + """ + own_copies: Final = tuple( + x + for x in model_list + if team_id is not None and _owner_team_id(x) == team_id and _team_public_model_name(x) == model_name ) + if own_copies: + return own_copies + by_name: Final = tuple(x for x in model_list if x.get("model_name") == model_name) + if by_name or team_id is not None: + return by_name + return tuple(x for x in model_list if _team_public_model_name(x) == model_name) + + +def deployments_targeted_by_name( + model_list: Sequence[Mapping[str, object]], model: str, team_id: str | None +) -> tuple[Mapping[str, object], ...]: + """``model`` targets deployments by ``litellm_params.model`` first, then the way a request for it routes.""" + by_param: Final = tuple(x for x in model_list if _deployment_model(x) == model) + return by_param or _deployments_routed_by_name(model_list, model, team_id) def _narrow_to_target( @@ -277,8 +310,7 @@ def _narrow_to_target( return by_id or tuple(model_list) if model is None: return tuple(model_list) - by_param: Final = tuple(x for x in model_list if _deployment_model(x) == model) - return by_param or tuple(x for x in model_list if deployment_answers_to(x, model, team_id)) + return deployments_targeted_by_name(model_list, model, team_id) def _is_strategy_router_deployment(litellm_params: Mapping[str, object]) -> bool: @@ -830,8 +862,10 @@ async def perform_health_check( When model_id is provided, only the deployment with that id is checked (so models that share the same name but have different ids are checked separately). - When model (name) is provided, all deployments matching that name are checked; - a team's public model name only matches for a caller from that team (``team_id``). + When model (name) is provided, the deployments a request for that name from the + caller (``team_id``) would route to are checked: the caller's team copies published + under that name, else the deployments named that way, else a public name that only + another team's deployment carries. When ``health_check_skip_disabled_background_models`` is True (via ``general_settings.health_check_skip_disabled_background_models``), deployments diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index dcef0fcd58a..002c4debb46 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -50,7 +50,7 @@ from litellm.proxy.health_check import ( ADMIN_ONLY_HEALTH_DISPLAY_PARAMS, _clean_endpoint_data, _update_litellm_params_for_health_check, - deployment_answers_to, + deployments_targeted_by_name, health_check_filter_kwargs_from_general_settings, perform_health_check, run_with_timeout, @@ -932,9 +932,9 @@ def _resolve_targeted_model_ids( Resolve a ``/health`` ``model`` / ``model_id`` query param to the set of deployment IDs the response should be scoped to, mirroring the live-path narrowing in ``perform_health_check()``: ``model_id`` wins when given and - matches ``model_info.id`` only; ``model`` matches the deployment's - ``model_name`` alias, its ``litellm_params.model`` provider string, or the - ``model_info.team_public_model_name`` the caller's own team reaches it by. + matches ``model_info.id`` only; ``model`` targets deployments by their + ``litellm_params.model`` provider string, else the deployments a request + for that name from the caller would route to (``deployments_targeted_by_name``). Callers pass an already-scoped list, so a ``model_id`` outside the caller's scope resolves to an empty set and never to the unvalidated id. @@ -946,9 +946,8 @@ def _resolve_targeted_model_ids( return None return { i - for m in model_list + for m in deployments_targeted_by_name(model_list, model, team_id) if (i := (m.get("model_info") or {}).get("id")) - and ((m.get("litellm_params") or {}).get("model") == model or deployment_answers_to(m, model, team_id)) } diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index c5950f6f30e..4c920dfb4d1 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -3069,7 +3069,7 @@ async def test_health_endpoint_returns_a_team_only_deployment_by_its_public_name @pytest.mark.asyncio -async def test_health_endpoint_targets_both_deployments_behind_a_shared_public_name_on_background_cache_path(): +async def test_health_endpoint_returns_only_the_owning_teams_copy_behind_a_shared_public_name_on_cache_path(): from fastapi import Response from litellm.proxy.health_endpoints._health_endpoints import health_endpoint @@ -3087,7 +3087,7 @@ async def test_health_endpoint_targets_both_deployments_behind_a_shared_public_n model_id=None, ) - assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-bedrock", "id-team-b"] + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-team-b"] async def _live_narrowed_model_ids( @@ -3134,14 +3134,15 @@ async def test_health_endpoint_keeps_an_admin_probe_by_name_off_other_teams_publ @pytest.mark.asyncio -async def test_health_endpoint_probes_both_deployments_behind_a_shared_public_name_for_the_owning_team(): +async def test_health_endpoint_probes_only_the_owning_teams_copy_behind_a_shared_public_name(): + """Team-b's requests for ``bedrock-nova`` route to its copy alone, so its health probe reaches only that copy.""" probed = await _live_narrowed_model_ids( _TEAM_MODEL_LIST, UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"], team_id="team-b"), model="bedrock-nova", ) - assert probed == {"id-bedrock", "id-team-b"} + assert probed == {"id-team-b"} @pytest.mark.asyncio @@ -3163,13 +3164,80 @@ async def test_health_endpoint_keeps_an_admin_probe_by_name_off_other_teams_publ assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-bedrock"] +@pytest.mark.asyncio +async def test_health_endpoint_probes_a_team_only_public_name_for_an_admin_on_live_path(): + """ + An admin's request for a public name only team-b's deployment carries routes + to that deployment, so the health probe for that name must reach it too + instead of answering an empty 503. + """ + probed = await _live_narrowed_model_ids(_TEAM_ONLY_MODEL_LIST, _ADMIN_OUTSIDE_TEAM_B, model="bedrock-nova") + + assert probed == {"id-team-b"} + + +@pytest.mark.asyncio +async def test_health_endpoint_returns_a_team_only_public_name_for_an_admin_on_background_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _TEAM_ONLY_MODEL_LIST, + _router_for(_TEAM_ONLY_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_TEAM_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), user_api_key_dict=_ADMIN_OUTSIDE_TEAM_B, model="bedrock-nova", model_id=None + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-team-b"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_background_health_checks", [False, True]) +async def test_health_endpoint_keeps_a_team_only_public_name_off_a_team_less_key(use_background_health_checks): + """ + A key with no team holds the name ``bedrock-nova`` but never sees team-b's + deployment, so the public-name fallback an admin gets must not open it up. + """ + from fastapi import HTTPException, Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with ( + _proxy_health_globals( + _TEAM_ONLY_MODEL_LIST, + _router_for(_TEAM_ONLY_MODEL_LIST), + use_background_health_checks=use_background_health_checks, + health_check_results=_TEAM_CACHED_RESULTS, + ), + patch( # test-quality-ok: the probe must never run; the endpoint has no injection seam for it + "litellm.proxy.health_endpoints._health_endpoints._perform_health_check_and_save", new_callable=AsyncMock + ) as probe, + pytest.raises(HTTPException) as refused, + ): + await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["bedrock-nova"]), + model="bedrock-nova", + model_id=None, + ) + + assert refused.value.status_code == 403 + assert "bedrock-nova" in str(refused.value.detail) + probe.assert_not_awaited() + + def test_resolve_targeted_model_ids_lets_model_id_win_over_model(): resolve = _health_endpoints_module._resolve_targeted_model_ids assert resolve(_TEAM_MODEL_LIST, "bedrock-nova", "id-team-b", None) == {"id-team-b"} assert resolve([_TEAM_MODEL_LIST[0]], "bedrock-nova", "id-team-b", None) == set() assert resolve(_TEAM_MODEL_LIST, "bedrock-nova", None, None) == {"id-bedrock"} - assert resolve(_TEAM_MODEL_LIST, "bedrock-nova", None, "team-b") == {"id-bedrock", "id-team-b"} + assert resolve(_TEAM_MODEL_LIST, "bedrock-nova", None, "team-b") == {"id-team-b"} + assert resolve(_TEAM_ONLY_MODEL_LIST, "bedrock-nova", None, None) == {"id-team-b"} @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index af20b80a718..e7abfcf126c 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -653,9 +653,8 @@ async def test_perform_health_check_narrows_to_a_team_deployment_by_its_public_n @pytest.mark.asyncio -@pytest.mark.parametrize("team_id", [None, "team-a"]) -async def test_perform_health_check_keeps_a_public_name_to_the_owning_team(team_id): - """A team's public model name is not a global alias: callers outside the team must not probe its deployment.""" +async def test_perform_health_check_keeps_a_public_name_off_another_team(): + """A team's public model name is not a global alias: a caller from another team must not probe its deployment.""" from litellm.proxy.health_check import perform_health_check team_deployment = { @@ -669,7 +668,7 @@ async def test_perform_health_check_keeps_a_public_name_to_the_owning_team(team_ "litellm.proxy.health_check._perform_health_check", probe ): healthy, unhealthy, _ = await perform_health_check( - model_list=[team_deployment], model="bedrock-nova", team_id=team_id + model_list=[team_deployment], model="bedrock-nova", team_id="team-a" ) probe.assert_not_awaited() @@ -677,6 +676,50 @@ async def test_perform_health_check_keeps_a_public_name_to_the_owning_team(team_ assert unhealthy == [] +_GLOBAL_DEPLOYMENT = { + "model_name": "bedrock-nova", + "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, + "model_info": {"id": "id-bedrock"}, +} +_TEAM_B_COPY = { + "model_name": "bedrock-nova_team-b_9f2c", + "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, + "model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("team_id", "model_list", "expected_ids"), + [ + (None, [_TEAM_B_COPY], ["id-team-b"]), + (None, [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock"]), + ("team-b", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-team-b"]), + ], + ids=[ + "a team-less caller reaches a public name nothing else carries", + "model_name wins over a public name for a team-less caller", + "a team's own copy wins over the global model_name", + ], +) +async def test_perform_health_check_targets_a_name_the_way_a_request_for_it_routes(team_id, model_list, expected_ids): + """``/health?model=`` probes the deployments a request for that name from the same caller would route to.""" + from litellm.proxy.health_check import perform_health_check + + probe = AsyncMock( + return_value=([{"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": i} for i in expected_ids], [], {}) + ) + + with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam + "litellm.proxy.health_check._perform_health_check", probe + ): + healthy, unhealthy, _ = await perform_health_check(model_list=model_list, model="bedrock-nova", team_id=team_id) + + assert [m["model_info"]["id"] for m in probe.call_args.args[0]] == expected_ids + assert [ep["model_id"] for ep in healthy] == expected_ids + assert unhealthy == [] + + def test_parse_background_health_check_model_groups_unset_returns_none(): from litellm.proxy.health_check import parse_background_health_check_model_groups From 5ab6e0858c48a4c954d8a3d11ee8eb1ab35a78e8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 11 Sep 2026 20:40:00 -0700 Subject: [PATCH 9/9] fix(health): resolve a model name the way a request routes before matching the provider model string --- litellm/proxy/health_check.py | 10 ++-- .../health_endpoints/_health_endpoints.py | 6 +-- .../health_endpoints/test_health_endpoints.py | 51 +++++++++++++++++++ .../proxy/test_health_check_functions.py | 30 ++++++++--- 4 files changed, 84 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index d774fd6e191..b1e4f6fd9c3 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -296,9 +296,10 @@ def _deployments_routed_by_name( def deployments_targeted_by_name( model_list: Sequence[Mapping[str, object]], model: str, team_id: str | None ) -> tuple[Mapping[str, object], ...]: - """``model`` targets deployments by ``litellm_params.model`` first, then the way a request for it routes.""" - by_param: Final = tuple(x for x in model_list if _deployment_model(x) == model) - return by_param or _deployments_routed_by_name(model_list, model, team_id) + """``model`` targets deployments the way a request for it routes, else by ``litellm_params.model``.""" + return _deployments_routed_by_name(model_list, model, team_id) or tuple( + x for x in model_list if _deployment_model(x) == model + ) def _narrow_to_target( @@ -865,7 +866,8 @@ async def perform_health_check( When model (name) is provided, the deployments a request for that name from the caller (``team_id``) would route to are checked: the caller's team copies published under that name, else the deployments named that way, else a public name that only - another team's deployment carries. + another team's deployment carries, else the deployments whose ``litellm_params.model`` + is that string. When ``health_check_skip_disabled_background_models`` is True (via ``general_settings.health_check_skip_disabled_background_models``), deployments diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 002c4debb46..1fc2f667b8b 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -932,9 +932,9 @@ def _resolve_targeted_model_ids( Resolve a ``/health`` ``model`` / ``model_id`` query param to the set of deployment IDs the response should be scoped to, mirroring the live-path narrowing in ``perform_health_check()``: ``model_id`` wins when given and - matches ``model_info.id`` only; ``model`` targets deployments by their - ``litellm_params.model`` provider string, else the deployments a request - for that name from the caller would route to (``deployments_targeted_by_name``). + matches ``model_info.id`` only; ``model`` targets the deployments a request + for that name from the caller would route to, else those whose + ``litellm_params.model`` provider string is that value (``deployments_targeted_by_name``). Callers pass an already-scoped list, so a ``model_id`` outside the caller's scope resolves to an empty set and never to the unvalidated id. diff --git a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py index 4c920dfb4d1..ca1e8a85396 100644 --- a/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/test_litellm/proxy/health_endpoints/test_health_endpoints.py @@ -3028,6 +3028,23 @@ async def test_health_endpoint_hides_team_deployments_from_a_key_with_no_team_on _TEAM_ONLY_MODEL_LIST = [_TEAM_MODEL_LIST[1]] +_BARE_NAME_MODEL_LIST = [ + {"model_name": "gpt-5.4-nano", "litellm_params": {"model": "gpt-5.4-nano"}, "model_info": {"id": "id-nano"}}, + { + "model_name": "gpt-5.4-nano_team-b_7c3d", + "litellm_params": {"model": "gpt-5.4-nano"}, + "model_info": {"id": "id-nano-team-b", "team_id": "team-b", "team_public_model_name": "gpt-5.4-nano"}, + }, +] +_BARE_NAME_CACHED_RESULTS = { + "healthy_endpoints": [ + {"model": "gpt-5.4-nano", "model_id": "id-nano"}, + {"model": "gpt-5.4-nano", "model_id": "id-nano-team-b"}, + ], + "unhealthy_endpoints": [], + "healthy_count": 2, + "unhealthy_count": 0, +} @pytest.mark.asyncio @@ -3145,6 +3162,40 @@ async def test_health_endpoint_probes_only_the_owning_teams_copy_behind_a_shared assert probed == {"id-team-b"} +@pytest.mark.asyncio +async def test_health_endpoint_probes_only_the_teams_copy_when_provider_model_equals_public_name(): + """A bare provider model equal to the public name must not pull the global copy into the team's probe.""" + probed = await _live_narrowed_model_ids( + _BARE_NAME_MODEL_LIST, + UserAPIKeyAuth(api_key="hashed-test-key", models=["gpt-5.4-nano"], team_id="team-b"), + model="gpt-5.4-nano", + ) + + assert probed == {"id-nano-team-b"} + + +@pytest.mark.asyncio +async def test_health_endpoint_returns_only_the_teams_copy_when_provider_model_equals_public_name_on_cache_path(): + from fastapi import Response + + from litellm.proxy.health_endpoints._health_endpoints import health_endpoint + + with _proxy_health_globals( + _BARE_NAME_MODEL_LIST, + _router_for(_BARE_NAME_MODEL_LIST), + use_background_health_checks=True, + health_check_results=_BARE_NAME_CACHED_RESULTS, + ): + result = await health_endpoint( + response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["gpt-5.4-nano"], team_id="team-b"), + model="gpt-5.4-nano", + model_id=None, + ) + + assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-nano-team-b"] + + @pytest.mark.asyncio async def test_health_endpoint_keeps_an_admin_probe_by_name_off_other_teams_public_copies_on_background_cache_path(): from fastapi import Response diff --git a/tests/test_litellm/proxy/test_health_check_functions.py b/tests/test_litellm/proxy/test_health_check_functions.py index e7abfcf126c..68fdeec2252 100644 --- a/tests/test_litellm/proxy/test_health_check_functions.py +++ b/tests/test_litellm/proxy/test_health_check_functions.py @@ -686,23 +686,41 @@ _TEAM_B_COPY = { "litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"}, "model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"}, } +_GLOBAL_BARE_NAME = { + "model_name": "gpt-5.4-nano", + "litellm_params": {"model": "gpt-5.4-nano"}, + "model_info": {"id": "id-nano"}, +} +_TEAM_B_BARE_COPY = { + "model_name": "gpt-5.4-nano_team-b_7c3d", + "litellm_params": {"model": "gpt-5.4-nano"}, + "model_info": {"id": "id-nano-team-b", "team_id": "team-b", "team_public_model_name": "gpt-5.4-nano"}, +} @pytest.mark.asyncio @pytest.mark.parametrize( - ("team_id", "model_list", "expected_ids"), + ("team_id", "model", "model_list", "expected_ids"), [ - (None, [_TEAM_B_COPY], ["id-team-b"]), - (None, [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock"]), - ("team-b", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-team-b"]), + (None, "bedrock-nova", [_TEAM_B_COPY], ["id-team-b"]), + (None, "bedrock-nova", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock"]), + ("team-b", "bedrock-nova", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-team-b"]), + ("team-b", "gpt-5.4-nano", [_GLOBAL_BARE_NAME, _TEAM_B_BARE_COPY], ["id-nano-team-b"]), + (None, "gpt-5.4-nano", [_GLOBAL_BARE_NAME, _TEAM_B_BARE_COPY], ["id-nano"]), + (None, "bedrock/us.amazon.nova-2-lite-v1:0", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock", "id-team-b"]), ], ids=[ "a team-less caller reaches a public name nothing else carries", "model_name wins over a public name for a team-less caller", "a team's own copy wins over the global model_name", + "a team's own copy wins over a litellm_params.model equal to the public name", + "model_name wins over a litellm_params.model equal to it for a team-less caller", + "a provider model string no name carries still matches litellm_params.model", ], ) -async def test_perform_health_check_targets_a_name_the_way_a_request_for_it_routes(team_id, model_list, expected_ids): +async def test_perform_health_check_targets_a_name_the_way_a_request_for_it_routes( + team_id, model, model_list, expected_ids +): """``/health?model=`` probes the deployments a request for that name from the same caller would route to.""" from litellm.proxy.health_check import perform_health_check @@ -713,7 +731,7 @@ async def test_perform_health_check_targets_a_name_the_way_a_request_for_it_rout with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam "litellm.proxy.health_check._perform_health_check", probe ): - healthy, unhealthy, _ = await perform_health_check(model_list=model_list, model="bedrock-nova", team_id=team_id) + healthy, unhealthy, _ = await perform_health_check(model_list=model_list, model=model, team_id=team_id) assert [m["model_info"]["id"] for m in probe.call_args.args[0]] == expected_ids assert [ep["model_id"] for ep in healthy] == expected_ids