mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
Merge pull request #40765 from BerriAI/litellm_fix_health_scoped_results_and_serialization
fix(proxy): expand access groups in /health scoping and allowlist health display fields
This commit is contained in:
commit
98d46ee59d
5 changed files with 1220 additions and 119 deletions
|
|
@ -32,32 +32,36 @@ from litellm.router_utils.auto_router_model_naming import (
|
|||
strategy_router_dependencies,
|
||||
)
|
||||
|
||||
ILLEGAL_DISPLAY_PARAMS: Final = [
|
||||
"messages",
|
||||
"api_key",
|
||||
"prompt",
|
||||
"input",
|
||||
"client_secret",
|
||||
"azure_ad_token",
|
||||
"azure_username",
|
||||
"azure_password",
|
||||
"vertex_credentials",
|
||||
"vertex_ai_credentials",
|
||||
"aws_access_key_id",
|
||||
"aws_secret_access_key",
|
||||
"aws_session_token",
|
||||
"aws_web_identity_token",
|
||||
"extra_headers",
|
||||
"headers",
|
||||
"exception", # internal; not JSON-serializable, never for display
|
||||
"litellm_metadata", # internal tracking metadata with auth objects; not for display
|
||||
]
|
||||
# Provider routing fields. Allowed for proxy admins so they can see which
|
||||
# region/version a deployment is checking; gated at the endpoint layer for
|
||||
# non-admin callers (see _strip_admin_only_fields_from_health_result).
|
||||
ADMIN_ONLY_HEALTH_DISPLAY_PARAMS: Final = ("api_base", "api_version")
|
||||
ADMIN_ONLY_HEALTH_DISPLAY_PARAMS: Final = ("api_base", "api_version", "aws_bedrock_runtime_endpoint")
|
||||
|
||||
MINIMAL_DISPLAY_PARAMS: Final = ["model", "mode_error"]
|
||||
MINIMAL_DISPLAY_PARAMS: Final = frozenset({"model", "mode_error"})
|
||||
|
||||
HEALTH_DISPLAY_PARAMS: Final = (
|
||||
MINIMAL_DISPLAY_PARAMS
|
||||
| frozenset(ADMIN_ONLY_HEALTH_DISPLAY_PARAMS)
|
||||
| frozenset(
|
||||
{
|
||||
"custom_llm_provider",
|
||||
"mode",
|
||||
"base_model",
|
||||
"aws_region_name",
|
||||
"region_name",
|
||||
"watsonx_region_name",
|
||||
"vertex_project",
|
||||
"vertex_location",
|
||||
"tpm",
|
||||
"rpm",
|
||||
"error",
|
||||
"raw_request_typed_dict",
|
||||
"x-ratelimit-remaining-requests",
|
||||
"x-ratelimit-remaining-tokens",
|
||||
"x-ms-region",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
# Modes whose health-check probe is a chat-style completion call and
|
||||
# therefore accept `max_tokens`. Other modes (embedding, image_generation,
|
||||
|
|
@ -143,14 +147,10 @@ def _get_random_llm_message():
|
|||
|
||||
def _clean_endpoint_data(endpoint_data: dict, details: bool | None = True):
|
||||
"""
|
||||
Clean the endpoint data for display to users.
|
||||
Keep only the explicitly approved, JSON-safe diagnostic fields for display to users.
|
||||
"""
|
||||
endpoint_data.pop("litellm_logging_obj", None)
|
||||
return (
|
||||
{k: v for k, v in endpoint_data.items() if k not in ILLEGAL_DISPLAY_PARAMS}
|
||||
if details is not False
|
||||
else {k: v for k, v in endpoint_data.items() if k in MINIMAL_DISPLAY_PARAMS}
|
||||
)
|
||||
displayed: Final = HEALTH_DISPLAY_PARAMS if details is not False else MINIMAL_DISPLAY_PARAMS
|
||||
return {k: v for k, v in endpoint_data.items() if k in displayed}
|
||||
|
||||
|
||||
def health_check_filter_kwargs_from_general_settings(
|
||||
|
|
@ -258,8 +258,52 @@ def _deployment_model(deployment: Mapping[str, object]) -> str | None:
|
|||
return params.get("model") if isinstance(params, Mapping) else None
|
||||
|
||||
|
||||
def _owner_team_id(deployment: Mapping[str, object]) -> str | None:
|
||||
info: Final = deployment.get("model_info")
|
||||
owner: Final = info.get("team_id") if isinstance(info, Mapping) else None
|
||||
return owner if isinstance(owner, str) else None
|
||||
|
||||
|
||||
def _team_public_model_name(deployment: Mapping[str, object]) -> str | None:
|
||||
info: Final = deployment.get("model_info")
|
||||
name: Final = info.get("team_public_model_name") if isinstance(info, Mapping) else None
|
||||
return name if isinstance(name, str) else None
|
||||
|
||||
|
||||
def _deployments_routed_by_name(
|
||||
model_list: Sequence[Mapping[str, object]], model_name: str, team_id: str | None
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
"""The deployments a request for ``model_name`` from this caller routes to.
|
||||
|
||||
A team's own copies published under that name win, then deployments carrying it as
|
||||
``model_name``. A caller with no team reaches a public name only when nothing carries
|
||||
it as ``model_name``, and only an admin still has another team's deployment in a
|
||||
scoped ``model_list`` by then.
|
||||
"""
|
||||
own_copies: Final = tuple(
|
||||
x
|
||||
for x in model_list
|
||||
if team_id is not None and _owner_team_id(x) == team_id and _team_public_model_name(x) == model_name
|
||||
)
|
||||
if own_copies:
|
||||
return own_copies
|
||||
by_name: Final = tuple(x for x in model_list if x.get("model_name") == model_name)
|
||||
if by_name or team_id is not None:
|
||||
return by_name
|
||||
return tuple(x for x in model_list if _team_public_model_name(x) == model_name)
|
||||
|
||||
|
||||
def deployments_targeted_by_name(
|
||||
model_list: Sequence[Mapping[str, object]], model: str, team_id: str | None
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
"""``model`` targets deployments the way a request for it routes, else by ``litellm_params.model``."""
|
||||
return _deployments_routed_by_name(model_list, model, team_id) or tuple(
|
||||
x for x in model_list if _deployment_model(x) == model
|
||||
)
|
||||
|
||||
|
||||
def _narrow_to_target(
|
||||
model_list: Sequence[Mapping[str, object]], model: str | None, model_id: str | None
|
||||
model_list: Sequence[Mapping[str, object]], model: str | None, model_id: str | None, team_id: str | None
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
"""Narrow to the requested deployment. An id matching nothing keeps the whole list."""
|
||||
if model_id is not None:
|
||||
|
|
@ -267,8 +311,7 @@ def _narrow_to_target(
|
|||
return by_id or tuple(model_list)
|
||||
if model is None:
|
||||
return tuple(model_list)
|
||||
by_param: Final = tuple(x for x in model_list if _deployment_model(x) == model)
|
||||
return by_param or tuple(x for x in model_list if x.get("model_name") == model)
|
||||
return deployments_targeted_by_name(model_list, model, team_id)
|
||||
|
||||
|
||||
def _is_strategy_router_deployment(litellm_params: Mapping[str, object]) -> bool:
|
||||
|
|
@ -813,13 +856,18 @@ async def perform_health_check(
|
|||
instrumentation_context: dict | None = None,
|
||||
health_check_skip_disabled_background_models: bool = False,
|
||||
router: "Router | None" = None,
|
||||
team_id: str | None = None,
|
||||
):
|
||||
"""
|
||||
Perform a health check on the system.
|
||||
|
||||
When model_id is provided, only the deployment with that id is checked
|
||||
(so models that share the same name but have different ids are checked separately).
|
||||
When model (name) is provided, all deployments matching that name are checked.
|
||||
When model (name) is provided, the deployments a request for that name from the
|
||||
caller (``team_id``) would route to are checked: the caller's team copies published
|
||||
under that name, else the deployments named that way, else a public name that only
|
||||
another team's deployment carries, else the deployments whose ``litellm_params.model``
|
||||
is that string.
|
||||
|
||||
When ``health_check_skip_disabled_background_models`` is True (via
|
||||
``general_settings.health_check_skip_disabled_background_models``), deployments
|
||||
|
|
@ -850,7 +898,7 @@ async def perform_health_check(
|
|||
cycle_start_time: Final = time.monotonic()
|
||||
requested_model_count: Final = len(model_list)
|
||||
skip_disabled: Final = health_check_skip_disabled_background_models
|
||||
narrowed: Final = _health_check_eligible(_narrow_to_target(model_list, model, model_id), skip_disabled)
|
||||
narrowed: Final = _health_check_eligible(_narrow_to_target(model_list, model, model_id, team_id), skip_disabled)
|
||||
if not narrowed:
|
||||
if instrumentation_enabled:
|
||||
logger.debug(
|
||||
|
|
|
|||
|
|
@ -36,9 +36,13 @@ from litellm.proxy._types import (
|
|||
UserAPIKeyAuth,
|
||||
WebhookEvent,
|
||||
)
|
||||
from litellm.proxy.auth.auth_checks import (
|
||||
_resolve_key_models_for_auth_check, # pyright: ignore[reportPrivateUsage] # the auth layer's sentinel resolution, reused so /health scopes exactly like a request
|
||||
)
|
||||
from litellm.proxy.auth.auth_utils import (
|
||||
_BANNED_REQUEST_BODY_PARAMS, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the request-body check
|
||||
)
|
||||
from litellm.proxy.auth.model_checks import get_key_models
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
||||
from litellm.proxy.db.health_check_latest import LatestHealthCheckRow
|
||||
|
|
@ -47,6 +51,7 @@ from litellm.proxy.health_check import (
|
|||
ADMIN_ONLY_HEALTH_DISPLAY_PARAMS,
|
||||
_clean_endpoint_data,
|
||||
_update_litellm_params_for_health_check,
|
||||
deployments_targeted_by_name,
|
||||
health_check_filter_kwargs_from_general_settings,
|
||||
perform_health_check,
|
||||
run_with_timeout,
|
||||
|
|
@ -58,6 +63,7 @@ from litellm.proxy.middleware.in_flight_requests_middleware import (
|
|||
get_in_flight_requests,
|
||||
)
|
||||
from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
|
||||
from litellm.router import Router
|
||||
from litellm.router_utils.clientside_credential_handler import (
|
||||
_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE, # pyright: ignore[reportPrivateUsage] # one canonical list, shared with the router path
|
||||
clientside_credential_keys,
|
||||
|
|
@ -917,7 +923,7 @@ def _is_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> bool:
|
|||
def _strip_admin_only_fields_from_health_result(result: dict) -> dict:
|
||||
"""
|
||||
Return a copy of the /health response with provider routing fields
|
||||
(``api_base``, ``api_version``) removed from each healthy/unhealthy
|
||||
(``ADMIN_ONLY_HEALTH_DISPLAY_PARAMS``) removed from each healthy/unhealthy
|
||||
endpoint entry. Used to hide those fields from non-admin callers while
|
||||
still showing them which deployments they own and whether each one is
|
||||
healthy. Proxy admins receive the unmodified result.
|
||||
|
|
@ -931,41 +937,68 @@ def _strip_admin_only_fields_from_health_result(result: dict) -> dict:
|
|||
return out
|
||||
|
||||
|
||||
def _resolve_targeted_model_ids(model_list: list, model: str | None, model_id: str | None) -> set | None:
|
||||
def _health_accessible_model_names(
|
||||
user_api_key_dict: UserAPIKeyAuth, llm_router: Router | None
|
||||
) -> frozenset[str] | None:
|
||||
"""Model names the caller may health-check, or None when the key is unrestricted."""
|
||||
granted_models: Final = _resolve_key_models_for_auth_check(user_api_key_dict)
|
||||
if not granted_models or SpecialModelNames.all_proxy_models.value in granted_models:
|
||||
return None
|
||||
if llm_router is None:
|
||||
return frozenset(granted_models)
|
||||
return frozenset(
|
||||
get_key_models(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
proxy_model_list=llm_router.get_model_names(team_id=user_api_key_dict.team_id),
|
||||
model_access_groups=llm_router.get_model_access_groups(),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _caller_may_probe_deployment(
|
||||
deployment: Mapping[str, object],
|
||||
allowed_models: frozenset[str] | None,
|
||||
llm_router: Router | None,
|
||||
team_id: str | None,
|
||||
caller_is_admin: bool,
|
||||
) -> bool:
|
||||
"""Same deployment visibility rule as routing: another team's deployment is never in scope, team-less callers included."""
|
||||
if not caller_is_admin and not Router._deployment_usable_by_team(deployment, team_id):
|
||||
return False
|
||||
if allowed_models is None:
|
||||
return True
|
||||
if llm_router is None:
|
||||
return deployment.get("model_name") in allowed_models
|
||||
model: Final = dict(deployment)
|
||||
return any(
|
||||
llm_router.should_include_deployment(model_name=name, model=model, team_id=team_id) for name in allowed_models
|
||||
)
|
||||
|
||||
|
||||
def _resolve_targeted_model_ids(
|
||||
model_list: list, model: str | None, model_id: str | None, team_id: str | None
|
||||
) -> set | None:
|
||||
"""
|
||||
Resolve a ``/health`` ``model`` / ``model_id`` query param to the set of
|
||||
deployment IDs the response should be scoped to.
|
||||
deployment IDs the response should be scoped to, mirroring the live-path
|
||||
narrowing in ``perform_health_check()``: ``model_id`` wins when given and
|
||||
matches ``model_info.id`` only; ``model`` targets the deployments a request
|
||||
for that name from the caller would route to, else those whose
|
||||
``litellm_params.model`` provider string is that value (``deployments_targeted_by_name``).
|
||||
|
||||
Mirrors the live-path semantics in ``perform_health_check()``: ``model``
|
||||
matches either the deployment's ``model_name`` alias or its
|
||||
``litellm_params.model`` provider string. ``model_id`` matches
|
||||
``model_info.id``.
|
||||
|
||||
Both query params are validated against the supplied ``model_list``.
|
||||
Callers pass an already-scoped list (filtered to the caller's allowed
|
||||
models for non-admins, full list for admins), so a ``model_id`` that
|
||||
isn't present resolves to an empty set rather than a single-element
|
||||
set — preventing a non-admin from reading another deployment's cached
|
||||
health entry by guessing its ID.
|
||||
|
||||
Returns ``None`` when no targeting is requested — callers should treat
|
||||
that as "no filter."
|
||||
Callers pass an already-scoped list, so a ``model_id`` outside the
|
||||
caller's scope resolves to an empty set and never to the unvalidated id.
|
||||
Returns ``None`` when no targeting is requested.
|
||||
"""
|
||||
if not model and not model_id:
|
||||
if model_id:
|
||||
return {i for m in model_list if (i := (m.get("model_info") or {}).get("id")) == model_id}
|
||||
if not model:
|
||||
return None
|
||||
target_ids: Final[set] = set()
|
||||
for m in model_list:
|
||||
deployment_id = (m.get("model_info") or {}).get("id")
|
||||
if not deployment_id:
|
||||
continue
|
||||
if model_id and deployment_id == model_id:
|
||||
target_ids.add(deployment_id)
|
||||
continue
|
||||
if model:
|
||||
litellm_model = (m.get("litellm_params") or {}).get("model")
|
||||
if m.get("model_name") == model or litellm_model == model:
|
||||
target_ids.add(deployment_id)
|
||||
return target_ids
|
||||
return {
|
||||
i
|
||||
for m in deployments_targeted_by_name(model_list, model, team_id)
|
||||
if (i := (m.get("model_info") or {}).get("id"))
|
||||
}
|
||||
|
||||
|
||||
def _filter_health_check_results_by_model_ids(results: dict, allowed_model_ids: set) -> dict:
|
||||
|
|
@ -1046,8 +1079,12 @@ def _health_endpoint_resolve_target_model_name(
|
|||
model_id: str | None,
|
||||
llm_router,
|
||||
) -> str | None:
|
||||
"""Map ``model_id`` (without ``model``) to ``model_name`` for live health checks."""
|
||||
if not model_id or model:
|
||||
"""Map ``model_id`` to its deployment's ``model_name`` for live health checks.
|
||||
|
||||
``model_id`` wins over ``model``, so an id no deployment carries is a 404 even
|
||||
when it is paired with a known name.
|
||||
"""
|
||||
if not model_id:
|
||||
return model
|
||||
if llm_router is None:
|
||||
raise HTTPException(
|
||||
|
|
@ -1133,7 +1170,9 @@ async def health_endpoint(
|
|||
response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
if is_admin:
|
||||
return result
|
||||
response.headers["Litellm-Health-Field-Notice"] = "api_base and api_version are admin-only on this endpoint"
|
||||
response.headers["Litellm-Health-Field-Notice"] = (
|
||||
f"{', '.join(ADMIN_ONLY_HEALTH_DISPLAY_PARAMS)} are admin-only on this endpoint"
|
||||
)
|
||||
return _strip_admin_only_fields_from_health_result(result)
|
||||
|
||||
try:
|
||||
|
|
@ -1157,32 +1196,24 @@ async def health_endpoint(
|
|||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail={"error": "Model list not initialized"},
|
||||
)
|
||||
_llm_model_list = copy.deepcopy(llm_model_list)
|
||||
### FILTER MODELS FOR ONLY THOSE USER HAS ACCESS TO ###
|
||||
# Live path: scope by model_name (every deployment has one).
|
||||
# Cache path: scope by model_id (the cache is keyed on model_id).
|
||||
# Consequence: a deployment whose model_name the caller can access
|
||||
# but which lacks model_info.id will appear in the live /health
|
||||
# response but NOT in the background-cache /health response. This is
|
||||
# surfaced via the "warnings" field below so operators can fix the
|
||||
# missing model_info.id rather than guess at the discrepancy.
|
||||
# Keys granted SpecialModelNames.all_proxy_models carry the literal
|
||||
# "all-proxy-models" entry, which matches no real model_name; treat
|
||||
# them as unrestricted instead of filtering the list down to nothing.
|
||||
# Keys granted SpecialModelNames.all_team_models inherit the parent
|
||||
# team's allowlist (same semantics as get_key_models in
|
||||
# model_checks.py). Without a team_id the sentinel cannot resolve and
|
||||
# stays in the list, matching nothing; denied rather than
|
||||
# unrestricted, mirroring _resolve_key_models_for_auth_check.
|
||||
accessible_models = list(user_api_key_dict.models)
|
||||
if SpecialModelNames.all_team_models.value in accessible_models and user_api_key_dict.team_id is not None:
|
||||
accessible_models = list(user_api_key_dict.team_models)
|
||||
restrict_to_allowed_models: Final = (
|
||||
len(accessible_models) > 0 and SpecialModelNames.all_proxy_models.value not in accessible_models
|
||||
)
|
||||
if restrict_to_allowed_models:
|
||||
allowed_models: Final = set(accessible_models)
|
||||
_llm_model_list = [m for m in _llm_model_list if m.get("model_name") in allowed_models]
|
||||
allowed_models: Final = _health_accessible_model_names(user_api_key_dict, llm_router)
|
||||
restrict_to_allowed_models: Final = not is_admin or allowed_models is not None
|
||||
_llm_model_list: Final = [
|
||||
m
|
||||
for m in copy.deepcopy(llm_model_list)
|
||||
if not restrict_to_allowed_models
|
||||
or _caller_may_probe_deployment(m, allowed_models, llm_router, user_api_key_dict.team_id, is_admin)
|
||||
]
|
||||
targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id, user_api_key_dict.team_id)
|
||||
if restrict_to_allowed_models and targeted_ids is not None and not targeted_ids:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail={
|
||||
"error": f"key not allowed to health-check model_id {model_id}"
|
||||
if model_id
|
||||
else f"key not allowed to health-check model {model}"
|
||||
},
|
||||
)
|
||||
if use_background_health_checks:
|
||||
# The cached background result covers every model. When the
|
||||
# caller targets a specific model/model_id we have to narrow the
|
||||
|
|
@ -1190,7 +1221,6 @@ async def health_endpoint(
|
|||
# healthy_count, otherwise an unhealthy "foo" combined with any
|
||||
# other healthy model would still report healthy_count > 0 and
|
||||
# the targeted-503 path would never fire.
|
||||
targeted_ids: Final = _resolve_targeted_model_ids(_llm_model_list, model, model_id)
|
||||
if restrict_to_allowed_models:
|
||||
allowed_model_ids: Final = {
|
||||
(m.get("model_info") or {}).get("id")
|
||||
|
|
@ -1202,7 +1232,7 @@ async def health_endpoint(
|
|||
# intersection of "targeted" and "allowed."
|
||||
filter_ids: Final = targeted_ids if targeted_ids is not None else allowed_model_ids
|
||||
filtered: Final = _filter_health_check_results_by_model_ids(health_check_results, filter_ids)
|
||||
if targeted_ids is None and not allowed_model_ids:
|
||||
if targeted_ids is None and _llm_model_list and not allowed_model_ids:
|
||||
# Caller has accessible model_names but none of the
|
||||
# matching deployments expose a model_info.id, so the
|
||||
# cache filter (which keys on model_id) drops every
|
||||
|
|
@ -1241,6 +1271,7 @@ async def health_endpoint(
|
|||
model_id=model_id,
|
||||
max_concurrency=health_check_concurrency,
|
||||
router=llm_router,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
**_hc_filter,
|
||||
)
|
||||
return _post_process(router_result)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -735,6 +735,121 @@ async def test_perform_health_check_and_save_forwards_skip_disabled_background_f
|
|||
assert call_kwargs["health_check_skip_disabled_background_models"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_perform_health_check_narrows_to_a_team_deployment_by_its_public_name():
|
||||
"""``/health?model=<team_public_model_name>`` must probe the team deployment, not an empty list."""
|
||||
from litellm.proxy.health_check import perform_health_check
|
||||
|
||||
team_deployment = {
|
||||
"model_name": "bedrock-nova_team-b_9f2c",
|
||||
"litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"},
|
||||
"model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"},
|
||||
}
|
||||
other_deployment = {
|
||||
"model_name": "gpt-5.4-mini",
|
||||
"litellm_params": {"model": "openai/gpt-5.4-mini"},
|
||||
"model_info": {"id": "id-openai"},
|
||||
}
|
||||
probe = AsyncMock(return_value=([{"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-team-b"}], [], {}))
|
||||
|
||||
with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam
|
||||
"litellm.proxy.health_check._perform_health_check", probe
|
||||
):
|
||||
healthy, unhealthy, _ = await perform_health_check(
|
||||
model_list=[team_deployment, other_deployment], model="bedrock-nova", team_id="team-b"
|
||||
)
|
||||
|
||||
assert [m["model_info"]["id"] for m in probe.call_args.args[0]] == ["id-team-b"]
|
||||
assert [ep["model_id"] for ep in healthy] == ["id-team-b"]
|
||||
assert unhealthy == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_perform_health_check_keeps_a_public_name_off_another_team():
|
||||
"""A team's public model name is not a global alias: a caller from another team must not probe its deployment."""
|
||||
from litellm.proxy.health_check import perform_health_check
|
||||
|
||||
team_deployment = {
|
||||
"model_name": "bedrock-nova_team-b_9f2c",
|
||||
"litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"},
|
||||
"model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"},
|
||||
}
|
||||
probe = AsyncMock(return_value=([{"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": "id-team-b"}], [], {}))
|
||||
|
||||
with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam
|
||||
"litellm.proxy.health_check._perform_health_check", probe
|
||||
):
|
||||
healthy, unhealthy, _ = await perform_health_check(
|
||||
model_list=[team_deployment], model="bedrock-nova", team_id="team-a"
|
||||
)
|
||||
|
||||
probe.assert_not_awaited()
|
||||
assert healthy == []
|
||||
assert unhealthy == []
|
||||
|
||||
|
||||
_GLOBAL_DEPLOYMENT = {
|
||||
"model_name": "bedrock-nova",
|
||||
"litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"},
|
||||
"model_info": {"id": "id-bedrock"},
|
||||
}
|
||||
_TEAM_B_COPY = {
|
||||
"model_name": "bedrock-nova_team-b_9f2c",
|
||||
"litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"},
|
||||
"model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"},
|
||||
}
|
||||
_GLOBAL_BARE_NAME = {
|
||||
"model_name": "gpt-5.4-nano",
|
||||
"litellm_params": {"model": "gpt-5.4-nano"},
|
||||
"model_info": {"id": "id-nano"},
|
||||
}
|
||||
_TEAM_B_BARE_COPY = {
|
||||
"model_name": "gpt-5.4-nano_team-b_7c3d",
|
||||
"litellm_params": {"model": "gpt-5.4-nano"},
|
||||
"model_info": {"id": "id-nano-team-b", "team_id": "team-b", "team_public_model_name": "gpt-5.4-nano"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("team_id", "model", "model_list", "expected_ids"),
|
||||
[
|
||||
(None, "bedrock-nova", [_TEAM_B_COPY], ["id-team-b"]),
|
||||
(None, "bedrock-nova", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock"]),
|
||||
("team-b", "bedrock-nova", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-team-b"]),
|
||||
("team-b", "gpt-5.4-nano", [_GLOBAL_BARE_NAME, _TEAM_B_BARE_COPY], ["id-nano-team-b"]),
|
||||
(None, "gpt-5.4-nano", [_GLOBAL_BARE_NAME, _TEAM_B_BARE_COPY], ["id-nano"]),
|
||||
(None, "bedrock/us.amazon.nova-2-lite-v1:0", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock", "id-team-b"]),
|
||||
],
|
||||
ids=[
|
||||
"a team-less caller reaches a public name nothing else carries",
|
||||
"model_name wins over a public name for a team-less caller",
|
||||
"a team's own copy wins over the global model_name",
|
||||
"a team's own copy wins over a litellm_params.model equal to the public name",
|
||||
"model_name wins over a litellm_params.model equal to it for a team-less caller",
|
||||
"a provider model string no name carries still matches litellm_params.model",
|
||||
],
|
||||
)
|
||||
async def test_perform_health_check_targets_a_name_the_way_a_request_for_it_routes(
|
||||
team_id, model, model_list, expected_ids
|
||||
):
|
||||
"""``/health?model=<name>`` probes the deployments a request for that name from the same caller would route to."""
|
||||
from litellm.proxy.health_check import perform_health_check
|
||||
|
||||
probe = AsyncMock(
|
||||
return_value=([{"model": "bedrock/us.amazon.nova-2-lite-v1:0", "model_id": i} for i in expected_ids], [], {})
|
||||
)
|
||||
|
||||
with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam
|
||||
"litellm.proxy.health_check._perform_health_check", probe
|
||||
):
|
||||
healthy, unhealthy, _ = await perform_health_check(model_list=model_list, model=model, team_id=team_id)
|
||||
|
||||
assert [m["model_info"]["id"] for m in probe.call_args.args[0]] == expected_ids
|
||||
assert [ep["model_id"] for ep in healthy] == expected_ids
|
||||
assert unhealthy == []
|
||||
|
||||
|
||||
def test_parse_background_health_check_model_groups_unset_returns_none():
|
||||
from litellm.proxy.health_check import parse_background_health_check_model_groups
|
||||
|
||||
|
|
|
|||
|
|
@ -872,9 +872,9 @@ def test_narrowing_by_an_id_that_matches_nothing_keeps_the_whole_list():
|
|||
"""Pinned because the disabled-dependency fix moved this filter into its own helper."""
|
||||
deployments = [{"model_name": "a", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a-1"}}]
|
||||
|
||||
assert hc_module._narrow_to_target(deployments, None, "no-such-id") == tuple(deployments)
|
||||
assert hc_module._narrow_to_target(deployments, None, "a-1") == tuple(deployments)
|
||||
assert hc_module._narrow_to_target(deployments, "a", None) == tuple(deployments)
|
||||
assert hc_module._narrow_to_target(deployments, None, "no-such-id", None) == tuple(deployments)
|
||||
assert hc_module._narrow_to_target(deployments, None, "a-1", None) == tuple(deployments)
|
||||
assert hc_module._narrow_to_target(deployments, "a", None, None) == tuple(deployments)
|
||||
|
||||
|
||||
def _nested_router_fixture(parent_tier: str):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue