fix(mcp): report reachability without stored credentials (#43240)

This commit is contained in:
tin-berri 2026-09-26 16:44:49 -07:00 • committed by GitHub
parent b2e82cf3be
commit 7244040908
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 644 additions and 143 deletions

View file

@ -73,9 +73,9 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
mcp_info: MCPInfo | None = None
static_headers: dict[str, str] | None = None
env_vars: list[MCPEnvVar] | None = None
status: Literal["healthy", "unhealthy", "unknown"] | None = Field(
status: Literal["healthy", "reachable", "unhealthy", "unknown"] | None = Field(
default="unknown",
description="Health status: 'healthy', 'unhealthy', 'unknown'",
description="Health status: 'healthy', 'unhealthy', 'unknown', or 'reachable' (requires include_reachability=true; authentication and tools unchecked)",
)
last_health_check: datetime | None = None
health_check_error: str | None = None

View file

@ -897,6 +897,41 @@ def _sanitized_error_text(exc: Exception) -> str:
return re.sub(r"https?://\S+", "<url>", str(exc))[:200]
async def _mcp_server_reachability(
server: MCPServer, *, timeout: float
) -> tuple[Literal["reachable", "unhealthy", "unknown"], str | None]:
if server.transport not in (MCPTransport.http, MCPTransport.sse) or not server.url:
return "unknown", "Server reachability requires an HTTP or SSE URL"
try:
url: Final = httpx.URL(server.url)
except (httpx.InvalidURL, ValueError):
return "unknown", "Server reachability requires an HTTP URL without embedded credentials"
if url.scheme not in ("http", "https") or not url.host or url.userinfo:
return "unknown", "Server reachability requires an HTTP URL without embedded credentials"
async def probe() -> None:
handler: Final = get_async_httpx_client(llm_provider="mcp_reachability")
async with handler.client.stream(
"GET",
url,
headers={"Accept": "text/event-stream, application/json"},
auth=None,
follow_redirects=False,
timeout=timeout,
):
pass
try:
await asyncio.wait_for(probe(), timeout=timeout)
except (asyncio.TimeoutError, httpx.TimeoutException):
return "unhealthy", f"Reachability check timed out after {timeout} seconds"
except asyncio.CancelledError:
return "unknown", "Reachability check was cancelled"
except Exception as exc:
return "unhealthy", f"Reachability check failed ({type(exc).__name__})"
return "reachable", None
async def _openapi_spec_health(
spec_path: str, *, timeout: float
) -> tuple[Literal["healthy", "unhealthy", "unknown"], str | None]:
@ -6986,13 +7021,9 @@ class MCPServerManager:
)
)
status: Literal["healthy", "unhealthy", "unknown"] = "unknown"
status: Literal["healthy", "reachable", "unhealthy", "unknown"] = "unknown"
health_check_error = None
# Check if we should skip health check based on auth configuration
should_skip_health_check = False
# Skip if server requires per-user authentication (OAuth2 or passthrough auth)
if (
server.requires_per_user_auth
or (
@ -7003,9 +7034,8 @@ class MCPServerManager:
)
or self._references_per_user_env_var(server)
):
should_skip_health_check = True
if not should_skip_health_check:
status, health_check_error = await _mcp_server_reachability(server, timeout=MCP_HEALTH_CHECK_TIMEOUT)
else:
try:
resolved_static_headers: Final = await self._resolve_static_headers_with_env_vars(
server=server,
@ -7081,6 +7111,8 @@ class MCPServerManager:
self,
user_api_key_auth: UserAPIKeyAuth | None = None,
server_ids: list[str] | None = None,
*,
checked_server_ids: frozenset[str] = frozenset(),
) -> list[LiteLLM_MCPServerTable]:
"""
Get all MCP servers that the user has access to, with health status and team information.
@ -7105,7 +7137,7 @@ class MCPServerManager:
# Check all accessible servers
target_server_ids = allowed_server_ids
return await self._run_health_checks(target_server_ids)
return await self._run_health_checks([sid for sid in target_server_ids if sid not in checked_server_ids])
async def get_all_allowed_mcp_servers(
self,
@ -7236,9 +7268,15 @@ class MCPServerManager:
if not target_server_ids:
return []
tasks: Final = [self.health_check_server(server_id) for server_id in target_server_ids]
results: Final = await asyncio.gather(*tasks)
return [server for server in results if server is not None]
unique_server_ids: Final = tuple(dict.fromkeys(target_server_ids))
batch_size: Final = 10
batches: Final = [
await asyncio.gather(
*(self.health_check_server(server_id) for server_id in unique_server_ids[offset : offset + batch_size])
)
for offset in range(0, len(unique_server_ids), batch_size)
]
return [server for batch in batches for server in batch if server is not None]
global_mcp_server_manager: Final[MCPServerManager] = MCPServerManager()

View file

@ -32168,6 +32168,7 @@
{
"enum": [
"healthy",
"reachable",
"unhealthy",
"unknown"
],
@ -32178,7 +32179,7 @@
}
],
"default": "unknown",
"description": "Health status: 'healthy', 'unhealthy', 'unknown'",
"description": "Health status: 'healthy', 'unhealthy', 'unknown', or 'reachable' (requires include_reachability=true; authentication and tools unchecked)",
"title": "Status"
},
"subject_token_type": {
@ -35224,6 +35225,7 @@
{
"enum": [
"healthy",
"reachable",
"unhealthy",
"unknown"
],
@ -35234,7 +35236,7 @@
}
],
"default": "unknown",
"description": "Health status: 'healthy', 'unhealthy', 'unknown'",
"description": "Health status: 'healthy', 'unhealthy', 'unknown', or 'reachable' (requires include_reachability=true; authentication and tools unchecked)",
"title": "Status"
},
"subject_token_type": {
@ -38095,6 +38097,18 @@
"description": "Server IDs to check. If not provided, checks all accessible servers.",
"title": "Server Ids"
}
},
{
"description": "Allow the 'reachable' status for responding servers whose authentication is unchecked.",
"in": "query",
"name": "include_reachability",
"required": false,
"schema": {
"default": false,
"description": "Allow the 'reachable' status for responding servers whose authentication is unchecked.",
"title": "Include Reachability",
"type": "boolean"
}
}
],
"responses": {
@ -38389,6 +38403,18 @@
"title": "Server Id",
"type": "string"
}
},
{
"description": "Allow the 'reachable' status for responding servers whose authentication is unchecked.",
"in": "query",
"name": "include_reachability",
"required": false,
"schema": {
"default": false,
"description": "Allow the 'reachable' status for responding servers whose authentication is unchecked.",
"title": "Include Reachability",
"type": "boolean"
}
}
],
"responses": {

View file

@ -1296,6 +1296,12 @@ if MCP_AVAILABLE:
return redacted_mcp_servers
def _mcp_health_status_for_response(
health_status: Literal["healthy", "reachable", "unhealthy", "unknown"] | None,
include_reachability: bool,
) -> Literal["healthy", "reachable", "unhealthy", "unknown"] | None:
return "unknown" if health_status == "reachable" and not include_reachability else health_status
@router.get(
"/server/health",
description="Health check for MCP servers",
@ -1307,6 +1313,10 @@ if MCP_AVAILABLE:
description="Server IDs to check. If not provided, checks all accessible servers.",
),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
include_reachability: Annotated[
bool,
Query(description="Allow the 'reachable' status for responding servers whose authentication is unchecked."),
] = False,
):
"""
Perform health checks on one or more MCP servers.
@ -1331,21 +1341,31 @@ if MCP_AVAILABLE:
if user_mcp_management_mode == "view_all" and not _is_restricted_virtual_key_request(user_api_key_dict):
servers = await global_mcp_server_manager.get_all_mcp_servers_with_health_unfiltered(server_ids=server_ids)
return [{"server_id": server.server_id, "status": server.status} for server in servers]
return [
{
"server_id": server.server_id,
"status": _mcp_health_status_for_response(server.status, include_reachability),
}
for server in servers
]
auth_contexts: Final = await build_effective_auth_contexts(user_api_key_dict)
server_status_map: Final[dict[str, Literal["healthy", "unhealthy", "unknown"] | None]] = {}
server_status_map: Final[dict[str, Literal["healthy", "reachable", "unhealthy", "unknown"] | None]] = {}
for auth_context in auth_contexts:
servers = await global_mcp_server_manager.get_all_mcp_servers_with_health_and_teams(
user_api_key_auth=auth_context,
server_ids=server_ids,
checked_server_ids=frozenset(server_status_map),
)
for server in servers:
if server.server_id not in server_status_map:
server_status_map[server.server_id] = server.status
return [{"server_id": server_id, "status": status} for server_id, status in server_status_map.items()]
return [
{"server_id": server_id, "status": _mcp_health_status_for_response(status, include_reachability)}
for server_id, status in server_status_map.items()
]
@router.post(
"/server/register",
@ -1615,6 +1635,10 @@ if MCP_AVAILABLE:
request: Request,
server_id: str,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
include_reachability: Annotated[
bool,
Query(description="Allow the 'reachable' status for responding servers whose authentication is unchecked."),
] = False,
):
"""
Get the info on the mcp server specified by the `server_id`
@ -1672,7 +1696,7 @@ if MCP_AVAILABLE:
try:
health_result: Final = await global_mcp_server_manager.health_check_server(server_id)
# Update the server object with health check results
mcp_server.status = health_result.status if health_result.status else "unknown"
mcp_server.status = _mcp_health_status_for_response(health_result.status, include_reachability) or "unknown"
mcp_server.last_health_check = health_result.last_health_check
mcp_server.health_check_error = health_result.health_check_error
except Exception as e:

View file

@ -6,7 +6,13 @@ connection. The DB-backed per-user flow is exercised in higher-level
tests in tests/mcp_tests.
"""
from typing import Final
from unittest.mock import AsyncMock
import pytest
from respx import MockRouter
from litellm.types.mcp_server.mcp_server_manager import MCPServer
# Look up these names lazily on every access. Tests in this directory call
# ``importlib.reload`` on the utils module to exercise registration logic,
@ -568,7 +574,7 @@ async def test_resolve_static_headers_user_value_wins_over_empty_global(
assert headers == {"Authorization": "Bearer user-secret"}
# ── health-check skip for per-user-env-var-backed headers ──────────────────
# ── health-check reachability for per-user-env-var-backed headers ───────────
@pytest.mark.parametrize(
@ -615,32 +621,26 @@ def test_references_per_user_env_var(static_headers, env_vars, expected):
@pytest.mark.asyncio
async def test_health_check_skips_servers_referencing_per_user_env_var(
mock_server, monkeypatch
):
"""A userless health probe cannot fill per-user ${NAME} placeholders, so a
server whose static_headers reference one must report 'unknown' without
connecting. Otherwise it forwards the literal placeholder upstream, gets a
401, and flips to 'unhealthy' even though real user calls succeed."""
async def test_health_check_reaches_servers_without_forwarding_per_user_env_vars(
mock_server: MCPServer, monkeypatch: pytest.MonkeyPatch, respx_mock: MockRouter
) -> None:
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
MCPServerManager,
)
manager = MCPServerManager()
manager: Final = MCPServerManager()
manager.registry[mock_server.server_id] = mock_server
create_client: Final = AsyncMock()
monkeypatch.setattr(manager, "_create_mcp_client", create_client)
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
route: Final = respx_mock.get(mock_server.url).respond(401)
created = []
result: Final = await manager.health_check_server(mock_server.server_id)
async def fake_create_client(*args, **kwargs):
created.append((args, kwargs))
raise RuntimeError("upstream rejected literal ${NAME}")
monkeypatch.setattr(manager, "_create_mcp_client", fake_create_client)
result = await manager.health_check_server(mock_server.server_id)
assert created == []
assert result.status == "unknown"
create_client.assert_not_called()
assert route.call_count == 1
assert not {"x-db-url", "x-other"}.intersection(route.calls[0].request.headers)
assert result.status == "reachable"
assert result.health_check_error is None

View file

@ -5,6 +5,7 @@ import json
import logging
import os
import sys
from collections.abc import AsyncIterator
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Final, Literal, Optional
@ -4894,69 +4895,258 @@ class TestMCPServerManager:
assert result.last_health_check is not None
@pytest.mark.asyncio
async def test_health_check_server_oauth2_skips_check(self):
"""Test that health check is skipped for OAuth2 servers and returns unknown status"""
manager = MCPServerManager()
# Mock OAuth2 server
server = MCPServer(
@pytest.mark.parametrize("oauth2_flow", [None, "authorization_code", "client_credentials"])
async def test_health_check_server_oauth2_reports_reachability(
self, monkeypatch: pytest.MonkeyPatch, respx_mock: MockRouter, oauth2_flow: Literal["authorization_code", "client_credentials"] | None
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
manager: Final = MCPServerManager()
server: Final = MCPServer(
server_id="oauth2-server",
name="oauth2-server",
transport=MCPTransport.http,
auth_type=MCPAuth.oauth2,
url="http://oauth2-server.com",
oauth2_flow=oauth2_flow,
client_id="client-id",
client_secret="stored-client-secret",
static_headers={"Authorization": "Bearer static-secret", "X-API-Key": "key-secret", "Cookie": "secret"},
)
manager.get_mcp_server_by_id = MagicMock(return_value=server)
# _create_mcp_client should not be called for OAuth2 servers
manager.registry[server.server_id] = server
manager._create_mcp_client = AsyncMock()
route: Final = respx_mock.get(server.url).respond(401)
# Perform health check
result = await manager.health_check_server("oauth2-server")
result: Final = await manager.health_check_server(server.server_id, mcp_auth_header="caller-secret")
# Verify that client was not created (health check was skipped)
manager._create_mcp_client.assert_not_called()
assert result.status == "reachable"
assert result.health_check_error is None
assert result.last_health_check is not None
assert route.call_count == 1
assert not {"authorization", "x-api-key", "cookie"}.intersection(route.calls[0].request.headers)
# Verify results
assert isinstance(result, LiteLLM_MCPServerTable)
assert result.server_id == "oauth2-server"
assert result.status == "unknown"
@pytest.mark.asyncio
@pytest.mark.parametrize("auth_type", [
MCPAuth.bearer_token, MCPAuth.api_key, MCPAuth.basic, MCPAuth.authorization, MCPAuth.token,
MCPAuth.oauth2_token_exchange, MCPAuth.oauth2_id_jag, MCPAuth.true_passthrough, MCPAuth.oauth_delegate,
])
@pytest.mark.parametrize("transport", [MCPTransport.http, MCPTransport.sse])
@pytest.mark.parametrize("response_code", [200, 204, 302, 401, 403, 405, 503])
async def test_health_check_without_credentials_accepts_any_http_response(
self, monkeypatch: pytest.MonkeyPatch, respx_mock: MockRouter, auth_type: MCPAuthType, transport: Literal[MCPTransport.http, MCPTransport.sse],
response_code: int,
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
manager: Final = MCPServerManager()
server: Final = MCPServer(
server_id="no-token-server",
name="no-token-server",
transport=transport,
auth_type=auth_type,
authentication_token=None,
url="http://no-token-server.com",
)
manager.registry[server.server_id] = server
manager._create_mcp_client = AsyncMock()
route: Final = respx_mock.get(server.url).respond(response_code)
result: Final = await manager.health_check_server(server.server_id)
manager._create_mcp_client.assert_not_called()
assert route.call_count == 1
assert result.status == "reachable"
assert result.health_check_error is None
assert result.last_health_check is not None
@pytest.mark.asyncio
async def test_health_check_server_no_token_skips_check(self):
"""Test that health check is skipped when auth_type is set but authentication_token is missing"""
manager = MCPServerManager()
@pytest.mark.parametrize("response_code", [200, 302])
async def test_health_reachability_closes_sse_without_body_redirect_or_cookie_reuse(
self, monkeypatch: pytest.MonkeyPatch, respx_mock: MockRouter, response_code: int
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
class UnreadBody(httpx.AsyncByteStream):
def __init__(self) -> None:
self.read = False
self.closed = False
# Mock server with auth_type but no authentication_token
server = MCPServer(
server_id="no-token-server",
name="no-token-server",
transport=MCPTransport.http,
auth_type=MCPAuth.bearer_token,
authentication_token=None, # No token
url="http://no-token-server.com",
async def __aiter__(self) -> AsyncIterator[bytes]:
self.read = True
yield b"secret SSE body"
async def aclose(self) -> None:
self.closed = True
manager: Final = MCPServerManager()
server: Final = MCPServer(
server_id="streaming-health", name="streaming-health", transport=MCPTransport.sse,
auth_type=MCPAuth.oauth2, url="https://mcp.example.test/events",
)
manager.registry[server.server_id] = server
bodies: Final = (UnreadBody(), UnreadBody())
route: Final = respx_mock.get(server.url).mock(side_effect=[
httpx.Response(response_code, stream=body, headers={
"Content-Type": "text/event-stream", "Set-Cookie": "health=secret; Path=/",
"Location": "http://127.0.0.1/private",
}) for body in bodies
])
first: Final = await manager.health_check_server(server.server_id)
second: Final = await manager.health_check_server(server.server_id)
assert (first.status, second.status) == ("reachable", "reachable")
assert route.call_count == len(respx_mock.calls) == 2
assert all(body.closed and not body.read for body in bodies)
assert all("cookie" not in call.request.headers for call in route.calls)
@pytest.mark.asyncio
@pytest.mark.parametrize(("transport", "url"), [
(MCPTransport.stdio, "https://mcp.example.test"),
(MCPTransport.http, None), (MCPTransport.http, ""), (MCPTransport.http, "not-a-url"),
(MCPTransport.http, "ftp://mcp.example.test"),
(MCPTransport.http, "https://user:secret@mcp.example.test"),
(MCPTransport.http, "https://mcp.example.test:bad/mcp"),
])
async def test_health_reachability_rejects_unprobeable_urls_without_requests(
self, respx_mock: MockRouter, transport: Literal[MCPTransport.http, MCPTransport.stdio], url: str | None
) -> None:
manager: Final = MCPServerManager()
server: Final = MCPServer(
server_id="unprobeable", name="unprobeable", transport=transport, auth_type=MCPAuth.oauth2, url=url,
)
manager.registry[server.server_id] = server
result: Final = await manager.health_check_server(server.server_id)
assert result.status == "unknown"
assert result.health_check_error and "secret" not in result.health_check_error
assert not respx_mock.calls
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", [
httpx.ConnectError("TLS/connection failure with secret details"),
httpx.ReadTimeout("secret timeout details"),
httpx.RemoteProtocolError("secret malformed response"),
])
async def test_health_reachability_reports_no_response_without_secret_details(
self, monkeypatch: pytest.MonkeyPatch, respx_mock: MockRouter, failure: httpx.RequestError
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
manager: Final = MCPServerManager()
server: Final = MCPServer(
server_id="failed-health", name="failed-health", transport=MCPTransport.http,
auth_type=MCPAuth.bearer_token, is_byok=True, url="https://mcp.example.test/secret?token=secret",
)
manager.registry[server.server_id] = server
route: Final = respx_mock.get(server.url).mock(side_effect=failure)
result: Final = await manager.health_check_server(server.server_id)
assert result.status == "unhealthy"
assert result.health_check_error and "secret" not in result.health_check_error
assert route.call_count == 1
@pytest.mark.asyncio
async def test_health_reachability_contains_ssl_setup_errors(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("SSL_SECURITY_LEVEL", "invalid-secret-cipher")
manager: Final = MCPServerManager()
server: Final = MCPServer(
server_id="bad-tls", name="bad-tls", transport=MCPTransport.http,
auth_type=MCPAuth.oauth2, url="https://mcp.example.test",
)
manager.registry[server.server_id] = server
result: Final = await manager.health_check_server(server.server_id)
assert result.status == "unhealthy"
assert result.health_check_error == "Reachability check failed (SSLError)"
@pytest.mark.asyncio
@pytest.mark.parametrize("cancel", [False, True])
async def test_health_reachability_timeout_and_cancellation_clean_up(
self, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch, cancel: bool
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
monkeypatch.setattr("litellm.proxy._experimental.mcp_server.mcp_server_manager.MCP_HEALTH_CHECK_TIMEOUT", 0.1)
manager: Final = MCPServerManager()
server: Final = MCPServer(
server_id="slow-health", name="slow-health", transport=MCPTransport.http,
auth_type=MCPAuth.oauth2, url="https://mcp.example.test/slow",
)
manager.registry[server.server_id] = server
started: Final = asyncio.Event()
stopped: Final = asyncio.Event()
async def slow_response(request: httpx.Request) -> httpx.Response:
started.set()
try:
await asyncio.Event().wait()
return httpx.Response(200)
finally:
stopped.set()
respx_mock.get(server.url).mock(side_effect=slow_response)
task: Final = asyncio.create_task(manager.health_check_server(server.server_id))
await asyncio.wait_for(started.wait(), timeout=1)
if cancel:
task.cancel()
result: Final = await task
assert result.status == ("unknown" if cancel else "unhealthy")
assert result.health_check_error == (
"Reachability check was cancelled" if cancel else "Reachability check timed out after 0.1 seconds"
)
assert stopped.is_set()
@pytest.mark.asyncio
@pytest.mark.parametrize("server_count", [0, 1, 10, 11, 25])
@pytest.mark.parametrize("filtered", [False, True])
async def test_bulk_health_checks_deduplicate_and_bound_upstream_requests(
self, monkeypatch: pytest.MonkeyPatch, respx_mock: MockRouter, server_count: int, filtered: bool
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
class Probe:
def __init__(self) -> None:
self.active = 0
self.peak = 0
async def respond(self, request: httpx.Request) -> httpx.Response:
self.active += 1
self.peak = max(self.peak, self.active)
try:
await asyncio.sleep(0)
return httpx.Response(401)
finally:
self.active -= 1
manager: Final = MCPServerManager()
server_ids: Final = [f"health-{index}" for index in range(server_count)]
manager.registry = {
server_id: MCPServer(
server_id=server_id, name=server_id, transport=MCPTransport.http,
auth_type=MCPAuth.oauth2, url=f"https://health.example.test/{server_id}",
)
for server_id in server_ids
}
probe: Final = Probe()
route: Final = respx_mock.get(host="health.example.test").mock(side_effect=probe.respond)
requested_ids: Final = [*server_ids, *reversed(server_ids), *server_ids, "not-registered"]
results: Final = (
await manager.get_all_mcp_servers_with_health_and_teams(
user_api_key_auth=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
server_ids=requested_ids,
)
if filtered
else await manager.get_all_mcp_servers_with_health_unfiltered(server_ids=requested_ids)
)
manager.get_mcp_server_by_id = MagicMock(return_value=server)
# _create_mcp_client should not be called
manager._create_mcp_client = AsyncMock()
# Perform health check
result = await manager.health_check_server("no-token-server")
# Verify that client was not created (health check was skipped)
manager._create_mcp_client.assert_not_called()
# Verify results
assert isinstance(result, LiteLLM_MCPServerTable)
assert result.server_id == "no-token-server"
assert result.status == "unknown"
assert result.health_check_error is None
assert result.last_health_check is not None
assert [(server.server_id, server.status) for server in results] == [
(server_id, "reachable") for server_id in server_ids
]
assert route.call_count == server_count
assert probe.peak == min(server_count, 10)
assert probe.active == 0
@pytest.mark.asyncio
async def test_health_check_server_with_static_headers(self):
@ -5003,70 +5193,58 @@ class TestMCPServerManager:
assert result.health_check_error is None
@pytest.mark.asyncio
async def test_health_check_skips_passthrough_auth_with_authorization_header(self):
"""Test that health check is skipped for servers with passthrough Authorization header"""
manager = MCPServerManager()
# Mock server with auth_type=none and Authorization in extra_headers (passthrough auth)
server = MCPServer(
async def test_health_check_reaches_passthrough_auth_with_authorization_header(
self, monkeypatch: pytest.MonkeyPatch, respx_mock: MockRouter
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
manager: Final = MCPServerManager()
server: Final = MCPServer(
server_id="github-server",
name="github-server",
transport=MCPTransport.http,
auth_type=MCPAuth.none,
authentication_token=None,
url="http://github-server.com",
extra_headers=["Authorization"], # Passthrough auth configured
extra_headers=["Authorization"],
)
manager.get_mcp_server_by_id = MagicMock(return_value=server)
# _create_mcp_client should not be called (health check should be skipped)
manager.registry[server.server_id] = server
manager._create_mcp_client = AsyncMock()
route: Final = respx_mock.get(server.url).respond(401)
# Perform health check
result = await manager.health_check_server("github-server")
result: Final = await manager.health_check_server(server.server_id)
# Verify that client was not created (health check was skipped)
manager._create_mcp_client.assert_not_called()
# Verify results
assert isinstance(result, LiteLLM_MCPServerTable)
assert result.server_id == "github-server"
assert result.status == "unknown"
assert route.call_count == 1
assert "authorization" not in route.calls[0].request.headers
assert result.status == "reachable"
assert result.health_check_error is None
assert result.last_health_check is not None
@pytest.mark.asyncio
async def test_health_check_skips_passthrough_auth_with_api_key_header(self):
"""Test that health check is skipped for servers with passthrough x-api-key header"""
manager = MCPServerManager()
# Mock server with auth_type=none and x-api-key in extra_headers
server = MCPServer(
async def test_health_check_reaches_passthrough_auth_with_api_key_header(
self, monkeypatch: pytest.MonkeyPatch, respx_mock: MockRouter
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
manager: Final = MCPServerManager()
server: Final = MCPServer(
server_id="sourcegraph-server",
name="sourcegraph-server",
transport=MCPTransport.http,
auth_type=MCPAuth.none,
authentication_token=None,
url="http://sourcegraph-server.com",
extra_headers=["x-api-key"], # Passthrough auth configured
extra_headers=["x-api-key"],
)
manager.get_mcp_server_by_id = MagicMock(return_value=server)
# _create_mcp_client should not be called
manager.registry[server.server_id] = server
manager._create_mcp_client = AsyncMock()
route: Final = respx_mock.get(server.url).respond(403)
# Perform health check
result = await manager.health_check_server("sourcegraph-server")
result: Final = await manager.health_check_server(server.server_id)
# Verify that client was not created (health check was skipped)
manager._create_mcp_client.assert_not_called()
# Verify results
assert isinstance(result, LiteLLM_MCPServerTable)
assert result.server_id == "sourcegraph-server"
assert result.status == "unknown"
assert route.call_count == 1
assert "x-api-key" not in route.calls[0].request.headers
assert result.status == "reachable"
assert result.health_check_error is None
assert result.last_health_check is not None
@ -9239,16 +9417,19 @@ class TestRegistryTableConversionPreservesEnvVars:
self._assert_env_vars_round_tripped(table)
@pytest.mark.asyncio
async def test_health_check_server_preserves_env_vars(self):
# OAuth2 without client credentials needs a per-user token, so the
# health check is skipped (no network) and we exercise the table
# construction path directly.
manager = MCPServerManager()
server = self._server_with_env_vars()
async def test_health_check_server_preserves_env_vars(
self, monkeypatch: pytest.MonkeyPatch, respx_mock: MockRouter
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
manager: Final = MCPServerManager()
server: Final = self._server_with_env_vars()
assert server.requires_per_user_auth is True
manager.registry[server.server_id] = server
table = await manager.health_check_server(server.server_id)
route: Final = respx_mock.get(server.url).respond(401)
table: Final = await manager.health_check_server(server.server_id)
self._assert_env_vars_round_tripped(table)
assert route.call_count == 1
assert "x-db-url" not in route.calls[0].request.headers
class TestHealthCheckInterpolatesGlobalEnvVars:

View file

@ -9,11 +9,12 @@ from contextlib import ExitStack, contextmanager
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from types import SimpleNamespace
from typing import Final, List, Optional, cast
from typing import Final, List, Literal, Optional, cast
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter, ValidationError
from respx import MockRouter
from fastapi import FastAPI, HTTPException
from fastapi.testclient import TestClient
@ -4311,6 +4312,170 @@ async def test_health_discovery_respects_route_restricted_key_grants(
assert all(row["status"] == expected_status for row in result)
@pytest.mark.asyncio
@pytest.mark.respx(assert_all_called=False)
@pytest.mark.parametrize("include_reachability", [False, True])
@pytest.mark.parametrize(
("requested", "expected"),
[
(None, ("shared", "first", "second")),
((), ("shared", "first", "second")),
(("shared", "shared", "denied"), ("shared",)),
(("second", "first"), ("first", "second")),
(("denied",), ()),
],
)
async def test_health_checks_probe_shared_servers_once_across_auth_contexts(
respx_mock: MockRouter,
monkeypatch: pytest.MonkeyPatch,
requested: tuple[str, ...] | None,
expected: tuple[str, ...],
include_reachability: bool,
) -> None:
from litellm.proxy._experimental.mcp_server import mcp_server_manager
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
manager: Final = mcp_server_manager.MCPServerManager()
manager.registry = {
server_id: MCPServer(
server_id=server_id,
name=server_id,
transport=MCPTransport.http,
auth_type=MCPAuth.oauth2,
url=f"https://mcp.example.test/{server_id}",
)
for server_id in ("shared", "first", "second", "denied")
}
routes: Final = {
server_id: respx_mock.get(server.url).respond(401)
for server_id, server in manager.registry.items()
}
contexts: Final = [
UserAPIKeyAuth(
user_role=LitellmUserRoles.INTERNAL_USER,
api_key=f"test-health-{index}",
object_permission=LiteLLM_ObjectPermissionTable(
object_permission_id=f"health-{index}", mcp_servers=list(grants)
),
)
for index, grants in enumerate((("shared", "first"), ("shared", "second")))
]
with (
patch.object(
mgmt_endpoints, "global_mcp_server_manager", manager
),
patch.object(
mcp_server_manager, "global_mcp_server_manager", manager
),
patch.object(
mgmt_endpoints, "build_effective_auth_contexts", AsyncMock(return_value=contexts)
),
patch("litellm.proxy.proxy_server.general_settings", {"user_mcp_management_mode": "restricted"}),
):
result: Final = await mgmt_endpoints.health_check_servers(
server_ids=list(requested) if requested is not None else None,
user_api_key_dict=contexts[0],
include_reachability=include_reachability,
)
expected_status: Final = "reachable" if include_reachability else "unknown"
assert sorted(result, key=lambda row: row["server_id"]) == [
{"server_id": server_id, "status": expected_status} for server_id in sorted(expected)
]
if requested:
assert [row["server_id"] for row in result] == list(expected)
assert {server_id: route.call_count for server_id, route in routes.items()} == {
server_id: int(server_id in expected) for server_id in routes
}
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["restricted", "view_all"])
@pytest.mark.parametrize("detail", [False, True])
@pytest.mark.parametrize("flag", [None, "false", "true"])
async def test_health_reachability_requires_explicit_api_opt_in(
respx_mock: MockRouter,
monkeypatch: pytest.MonkeyPatch,
mode: str,
detail: bool,
flag: str | None,
) -> None:
from litellm.proxy._experimental.mcp_server import mcp_server_manager
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
class HealthResponse(BaseModel):
server_id: str
status: str | None
class LegacyHealthResponse(BaseModel):
server_id: str
status: Literal["healthy", "unhealthy", "unknown"] | None
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
manager: Final = mcp_server_manager.MCPServerManager()
server: Final = MCPServer(
server_id="health-compatibility",
name="health-compatibility",
transport=MCPTransport.http,
auth_type=MCPAuth.oauth2,
url="https://mcp.example.test/mcp",
)
manager.registry[server.server_id] = server
route: Final = respx_mock.get(server.url).respond(401)
caller: Final = UserAPIKeyAuth(
user_role=LitellmUserRoles.PROXY_ADMIN,
api_key="test-health-compatibility",
object_permission=LiteLLM_ObjectPermissionTable(
object_permission_id="health-compatibility", mcp_servers=[server.server_id]
),
)
def authenticated_caller() -> UserAPIKeyAuth:
return caller
app: Final = FastAPI()
app.include_router(mgmt_endpoints.router)
app.dependency_overrides[mgmt_endpoints.user_api_key_auth] = authenticated_caller
suffix: Final = server.server_id if detail else "health"
query: Final = {} if flag is None else {"include_reachability": flag}
with (
patch.object( # test-quality-ok: TQ008 inject the real registry into the legacy route binding
mgmt_endpoints, "global_mcp_server_manager", manager
),
patch.object( # test-quality-ok: TQ008 permission resolution uses the shared registry
mcp_server_manager, "global_mcp_server_manager", manager
),
patch("litellm.proxy.proxy_server.general_settings", {"user_mcp_management_mode": mode}),
patch.object( # test-quality-ok: TQ008 select the config-backed detail path without a database
mgmt_endpoints, "get_prisma_client_or_throw", return_value=MagicMock()
),
patch.object( # test-quality-ok: TQ008 a missing database row falls back to the real registry
mgmt_endpoints, "get_mcp_server", AsyncMock(return_value=None)
),
):
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://gateway") as client:
response: Final = await client.get(f"/v1/mcp/server/{suffix}", params=query)
assert response.status_code == 200, response.text
rows: Final = (
[HealthResponse.model_validate_json(response.content)]
if detail else TypeAdapter(list[HealthResponse]).validate_json(response.content)
)
expected_status: Final = "reachable" if flag == "true" else "unknown"
assert [row.model_dump() for row in rows] == [{"server_id": server.server_id, "status": expected_status}]
assert route.call_count == 1
legacy_parser: Final = (
LegacyHealthResponse.model_validate_json
if detail else TypeAdapter(list[LegacyHealthResponse]).validate_json
)
if flag == "true":
with pytest.raises(ValidationError, match="literal_error"):
legacy_parser(response.content)
else:
legacy_parser(response.content)
class TestMCPRegistryEndpoint:
def test_registry_returns_404_when_flag_missing(self):
client = create_mcp_router_test_client()

View file

@ -1,5 +1,6 @@
import React from "react";
import { fireEvent, render, screen } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { describe, it, expect, vi, afterEach } from "vitest";
import MCPServerCard from "./MCPServerCard";
import type { MCPServer } from "@/components/mcp_tools/types";
@ -18,6 +19,19 @@ function renderCard(overrides: Partial<MCPServer>) {
render(<MCPServerCard server={{ ...baseServer, ...overrides } as MCPServer} onClick={vi.fn()} />);
}
describe("MCPServerCard health", () => {
it("explains that reachable does not verify authentication or tools", async () => {
const user = userEvent.setup();
renderCard({ status: "reachable", oauth2_flow: "authorization_code" });
await user.hover(screen.getByText("Reachable"));
expect(await screen.findByText("Server responded. Authentication and tools were not checked")).toBeInTheDocument();
expect(screen.queryByText("No health data")).not.toBeInTheDocument();
expect(screen.queryByText("Healthy")).not.toBeInTheDocument();
});
});
describe("MCPServerCard OAuth flow indicator", () => {
it("shows the 'OAuth flow not set' badge for an oauth2 server with no oauth2_flow", () => {
renderCard({ auth_type: "oauth2", oauth2_flow: null });

View file

@ -11,7 +11,7 @@ import {
} from "@/components/ui/dropdown-menu";
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
import { cn } from "@/lib/cva.config";
import { AUTH_TYPE, type MCPServer } from "@/components/mcp_tools/types";
import { AUTH_TYPE, MCP_REACHABLE_DESCRIPTION, type MCPServer } from "@/components/mcp_tools/types";
import { Logo } from "@/components/molecules/logo/Logo";
import { getMaskedAndFullUrl } from "./utils";
@ -33,6 +33,7 @@ interface MCPServerCardProps {
const HEALTH_TONE: Record<string, { dot: string }> = {
healthy: { dot: "bg-success" },
reachable: { dot: "bg-info" },
unhealthy: { dot: "bg-destructive" },
unknown: { dot: "bg-border" },
};
@ -332,6 +333,7 @@ const HealthChip: FC<HealthChipProps> = ({
</Badge>
);
}
const hasHealthData = Boolean(lastCheck || error || status === "reachable");
return (
<Tooltip>
<TooltipTrigger
@ -355,6 +357,7 @@ const HealthChip: FC<HealthChipProps> = ({
/>
<TooltipContent side="top" className="max-w-xs">
<div className="mb-1 font-semibold">Health: {status}</div>
{status === "reachable" && <div className="mb-1 text-xs">{MCP_REACHABLE_DESCRIPTION}</div>}
{lastCheck && <div className="mb-1 text-xs">Last check: {new Date(lastCheck).toLocaleString()}</div>}
{error && (
<div className="text-xs">
@ -362,7 +365,7 @@ const HealthChip: FC<HealthChipProps> = ({
<div className="wrap-break-word">{error}</div>
</div>
)}
{!lastCheck && !error && <div className="text-xs">No health data</div>}
{!hasHealthData && <div className="text-xs">No health data</div>}
{onRecheck && <div className="mt-1 text-xs">Click to recheck</div>}
</TooltipContent>
</Tooltip>

View file

@ -113,12 +113,14 @@ describe("compareServers", () => {
it("sorts health before recency and display name", () => {
const servers: MCPServer[] = [
{ ...server("healthy", "aaa", "2026-03-01T00:00:00Z"), status: "healthy" },
{ ...server("reachable", "aaa", "2026-04-01T00:00:00Z"), status: "reachable" },
{ ...server("unknown", "bbb", "2026-02-01T00:00:00Z"), status: "unknown" },
{ ...server("unhealthy", "zzz", "2026-01-01T00:00:00Z"), status: "unhealthy" },
];
expect(servers.sort((a, b) => compareServers(a, b, "health")).map((s) => s.server_id)).toEqual([
"unhealthy",
"unknown",
"reachable",
"healthy",
]);
});

View file

@ -61,7 +61,8 @@ const SORT_OPTIONS: { value: SortKey; label: string }[] = [
const HEALTH_RANK: Record<string, number> = {
unhealthy: 0,
unknown: 1,
healthy: 2,
reachable: 2,
healthy: 3,
};
const compareByName = (a: MCPServer, b: MCPServer): number => {
@ -191,7 +192,7 @@ const MCPServers: React.FC<MCPServerProps> = ({ accessToken, userRole, userID, i
const healthStatus = healthMap.get(server.server_id);
return {
...server,
status: healthStatus ? (healthStatus as "healthy" | "unhealthy" | "unknown") : server.status,
status: healthStatus ? (healthStatus as MCPServer["status"]) : server.status,
};
});
}, [mcpServers, healthStatuses]);

View file

@ -28,10 +28,10 @@ const mockServer: MCPServerData = {
env: {},
};
function renderTable(onServerClick = vi.fn()) {
function renderTable(onServerClick = vi.fn(), servers = [mockServer]) {
render(
<DataTable
data={[mockServer]}
data={servers}
columns={getMCPHubTableColumns({ onServerClick })}
getRowId={(server) => server.server_id}
sortingMode="client"
@ -42,6 +42,15 @@ function renderTable(onServerClick = vi.fn()) {
}
describe("getMCPHubTableColumns", () => {
it("explains the limited check for a reachable server", async () => {
const user = userEvent.setup();
renderTable(vi.fn(), [{ ...mockServer, status: "reachable" }]);
await user.hover(screen.getByText("reachable"));
expect(await screen.findByText("Server responded. Authentication and tools were not checked")).toBeInTheDocument();
});
it("renders the server row", () => {
renderTable();
expect(screen.getByText("exa_test")).toBeInTheDocument();

View file

@ -4,6 +4,7 @@ import { ColumnDef } from "@tanstack/react-table";
import { Copy, Info, MoreHorizontal } from "lucide-react";
import { DataTableSortHeader } from "@/components/shared/DataTable";
import { MCP_REACHABLE_DESCRIPTION } from "@/components/mcp_tools/types";
import { IdentityCell, StatusBadge, type StatusTone } from "@/components/shared/table_cells";
import { Badge } from "@/components/ui/badge";
import { buttonVariants } from "@/components/ui/button";
@ -49,6 +50,7 @@ const STATUS_TONES: Record<string, StatusTone> = {
inactive: "error",
unknown: "neutral",
healthy: "success",
reachable: "info",
unhealthy: "error",
};
@ -150,7 +152,11 @@ export const getMCPHubTableColumns = ({ onServerClick }: MCPHubTableColumnsDeps)
enableSorting: true,
sortingFn: "alphanumeric",
cell: ({ row }) => (
<StatusBadge tone={STATUS_TONES[row.original.status] || "neutral"} label={row.original.status || "unknown"} />
<StatusBadge
tone={STATUS_TONES[row.original.status] || "neutral"}
label={row.original.status || "unknown"}
tooltip={row.original.status === "reachable" ? MCP_REACHABLE_DESCRIPTION : undefined}
/>
),
},
{

View file

@ -405,6 +405,8 @@ export interface MCPToolsViewerProps {
extraHeaders?: string[] | null;
}
export const MCP_REACHABLE_DESCRIPTION = "Server responded. Authentication and tools were not checked";
export interface MCPServer {
server_id: string;
is_config?: boolean;
@ -435,7 +437,7 @@ export interface MCPServer {
updated_by: string;
extra_headers?: string[] | null;
static_headers?: Record<string, string> | null;
status?: "healthy" | "unhealthy" | "unknown";
status?: "healthy" | "reachable" | "unhealthy" | "unknown";
last_health_check?: string | null;
health_check_error?: string | null;
teams?: Team[];

View file

@ -706,6 +706,30 @@ describe("testMCPToolsListRequest auth headers", () => {
});
});
describe("fetchMCPServerHealth", () => {
const originalFetch = global.fetch;
afterEach(() => {
global.fetch = originalFetch;
});
it.each([{ serverIds: undefined }, { serverIds: [] }, { serverIds: ["server one", "server&two"] }])(
"opts into reachability while preserving requested servers: $serverIds",
async ({ serverIds }) => {
const mockFetch = vi.fn<typeof fetch>().mockResolvedValue(new Response("[]", { status: 200 }));
global.fetch = mockFetch;
await Networking.fetchMCPServerHealth("test-token", serverIds);
expect(mockFetch).toHaveBeenCalledOnce();
const url = new URL(String(mockFetch.mock.calls[0][0]), "http://localhost");
expect(url.pathname).toMatch(/\/v1\/mcp\/server\/health$/);
expect(url.searchParams.get("include_reachability")).toBe("true");
expect(url.searchParams.getAll("server_ids")).toEqual(serverIds ?? []);
},
);
});
describe("getAutoRouterClassifierDefaultPromptCall", () => {
const originalFetch = global.fetch;

View file

@ -4968,6 +4968,7 @@ export const fetchMCPServerHealth = async (accessToken: string, serverIds?: stri
return await apiClient.get(`/v1/mcp/server/health`, {
accessToken,
query: {
include_reachability: true,
server_ids: serverIds && serverIds.length > 0 ? serverIds : undefined,
},
});

View file

@ -32504,10 +32504,10 @@ export interface components {
} | null;
/**
* Status
* @description Health status: 'healthy', 'unhealthy', 'unknown'
* @description Health status: 'healthy', 'unhealthy', 'unknown', or 'reachable' (requires include_reachability=true; authentication and tools unchecked)
* @default unknown
*/
status: ("healthy" | "unhealthy" | "unknown") | null;
status: ("healthy" | "reachable" | "unhealthy" | "unknown") | null;
/** Subject Token Type */
subject_token_type?: string | null;
/** Submitted At */
@ -72212,6 +72212,8 @@ export interface operations {
query?: {
/** @description Server IDs to check. If not provided, checks all accessible servers. */
server_ids?: string[] | null;
/** @description Allow the 'reachable' status for responding servers whose authentication is unchecked. */
include_reachability?: boolean;
};
header?: never;
path?: never;
@ -72363,7 +72365,10 @@ export interface operations {
};
fetch_mcp_server_v1_mcp_server__server_id__get: {
parameters: {
query?: never;
query?: {
/** @description Allow the 'reachable' status for responding servers whose authentication is unchecked. */
include_reachability?: boolean;
};
header?: never;
path: {
server_id: string;