mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
fix(health): resolve a model name the way a request routes before matching the provider model string
This commit is contained in:
parent
eaf3d8ad3e
commit
5ab6e0858c
4 changed files with 84 additions and 13 deletions
|
|
@ -296,9 +296,10 @@ def _deployments_routed_by_name(
|
|||
def deployments_targeted_by_name(
|
||||
model_list: Sequence[Mapping[str, object]], model: str, team_id: str | None
|
||||
) -> tuple[Mapping[str, object], ...]:
|
||||
"""``model`` targets deployments by ``litellm_params.model`` first, then the way a request for it routes."""
|
||||
by_param: Final = tuple(x for x in model_list if _deployment_model(x) == model)
|
||||
return by_param or _deployments_routed_by_name(model_list, model, team_id)
|
||||
"""``model`` targets deployments the way a request for it routes, else by ``litellm_params.model``."""
|
||||
return _deployments_routed_by_name(model_list, model, team_id) or tuple(
|
||||
x for x in model_list if _deployment_model(x) == model
|
||||
)
|
||||
|
||||
|
||||
def _narrow_to_target(
|
||||
|
|
@ -865,7 +866,8 @@ async def perform_health_check(
|
|||
When model (name) is provided, the deployments a request for that name from the
|
||||
caller (``team_id``) would route to are checked: the caller's team copies published
|
||||
under that name, else the deployments named that way, else a public name that only
|
||||
another team's deployment carries.
|
||||
another team's deployment carries, else the deployments whose ``litellm_params.model``
|
||||
is that string.
|
||||
|
||||
When ``health_check_skip_disabled_background_models`` is True (via
|
||||
``general_settings.health_check_skip_disabled_background_models``), deployments
|
||||
|
|
|
|||
|
|
@ -932,9 +932,9 @@ def _resolve_targeted_model_ids(
|
|||
Resolve a ``/health`` ``model`` / ``model_id`` query param to the set of
|
||||
deployment IDs the response should be scoped to, mirroring the live-path
|
||||
narrowing in ``perform_health_check()``: ``model_id`` wins when given and
|
||||
matches ``model_info.id`` only; ``model`` targets deployments by their
|
||||
``litellm_params.model`` provider string, else the deployments a request
|
||||
for that name from the caller would route to (``deployments_targeted_by_name``).
|
||||
matches ``model_info.id`` only; ``model`` targets the deployments a request
|
||||
for that name from the caller would route to, else those whose
|
||||
``litellm_params.model`` provider string is that value (``deployments_targeted_by_name``).
|
||||
|
||||
Callers pass an already-scoped list, so a ``model_id`` outside the
|
||||
caller's scope resolves to an empty set and never to the unvalidated id.
|
||||
|
|
|
|||
|
|
@ -3028,6 +3028,23 @@ async def test_health_endpoint_hides_team_deployments_from_a_key_with_no_team_on
|
|||
|
||||
|
||||
_TEAM_ONLY_MODEL_LIST = [_TEAM_MODEL_LIST[1]]
|
||||
_BARE_NAME_MODEL_LIST = [
|
||||
{"model_name": "gpt-5.4-nano", "litellm_params": {"model": "gpt-5.4-nano"}, "model_info": {"id": "id-nano"}},
|
||||
{
|
||||
"model_name": "gpt-5.4-nano_team-b_7c3d",
|
||||
"litellm_params": {"model": "gpt-5.4-nano"},
|
||||
"model_info": {"id": "id-nano-team-b", "team_id": "team-b", "team_public_model_name": "gpt-5.4-nano"},
|
||||
},
|
||||
]
|
||||
_BARE_NAME_CACHED_RESULTS = {
|
||||
"healthy_endpoints": [
|
||||
{"model": "gpt-5.4-nano", "model_id": "id-nano"},
|
||||
{"model": "gpt-5.4-nano", "model_id": "id-nano-team-b"},
|
||||
],
|
||||
"unhealthy_endpoints": [],
|
||||
"healthy_count": 2,
|
||||
"unhealthy_count": 0,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -3145,6 +3162,40 @@ async def test_health_endpoint_probes_only_the_owning_teams_copy_behind_a_shared
|
|||
assert probed == {"id-team-b"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_endpoint_probes_only_the_teams_copy_when_provider_model_equals_public_name():
|
||||
"""A bare provider model equal to the public name must not pull the global copy into the team's probe."""
|
||||
probed = await _live_narrowed_model_ids(
|
||||
_BARE_NAME_MODEL_LIST,
|
||||
UserAPIKeyAuth(api_key="hashed-test-key", models=["gpt-5.4-nano"], team_id="team-b"),
|
||||
model="gpt-5.4-nano",
|
||||
)
|
||||
|
||||
assert probed == {"id-nano-team-b"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_endpoint_returns_only_the_teams_copy_when_provider_model_equals_public_name_on_cache_path():
|
||||
from fastapi import Response
|
||||
|
||||
from litellm.proxy.health_endpoints._health_endpoints import health_endpoint
|
||||
|
||||
with _proxy_health_globals(
|
||||
_BARE_NAME_MODEL_LIST,
|
||||
_router_for(_BARE_NAME_MODEL_LIST),
|
||||
use_background_health_checks=True,
|
||||
health_check_results=_BARE_NAME_CACHED_RESULTS,
|
||||
):
|
||||
result = await health_endpoint(
|
||||
response=Response(),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-test-key", models=["gpt-5.4-nano"], team_id="team-b"),
|
||||
model="gpt-5.4-nano",
|
||||
model_id=None,
|
||||
)
|
||||
|
||||
assert [ep["model_id"] for ep in result["healthy_endpoints"]] == ["id-nano-team-b"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_health_endpoint_keeps_an_admin_probe_by_name_off_other_teams_public_copies_on_background_cache_path():
|
||||
from fastapi import Response
|
||||
|
|
|
|||
|
|
@ -686,23 +686,41 @@ _TEAM_B_COPY = {
|
|||
"litellm_params": {"model": "bedrock/us.amazon.nova-2-lite-v1:0"},
|
||||
"model_info": {"id": "id-team-b", "team_id": "team-b", "team_public_model_name": "bedrock-nova"},
|
||||
}
|
||||
_GLOBAL_BARE_NAME = {
|
||||
"model_name": "gpt-5.4-nano",
|
||||
"litellm_params": {"model": "gpt-5.4-nano"},
|
||||
"model_info": {"id": "id-nano"},
|
||||
}
|
||||
_TEAM_B_BARE_COPY = {
|
||||
"model_name": "gpt-5.4-nano_team-b_7c3d",
|
||||
"litellm_params": {"model": "gpt-5.4-nano"},
|
||||
"model_info": {"id": "id-nano-team-b", "team_id": "team-b", "team_public_model_name": "gpt-5.4-nano"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("team_id", "model_list", "expected_ids"),
|
||||
("team_id", "model", "model_list", "expected_ids"),
|
||||
[
|
||||
(None, [_TEAM_B_COPY], ["id-team-b"]),
|
||||
(None, [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock"]),
|
||||
("team-b", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-team-b"]),
|
||||
(None, "bedrock-nova", [_TEAM_B_COPY], ["id-team-b"]),
|
||||
(None, "bedrock-nova", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock"]),
|
||||
("team-b", "bedrock-nova", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-team-b"]),
|
||||
("team-b", "gpt-5.4-nano", [_GLOBAL_BARE_NAME, _TEAM_B_BARE_COPY], ["id-nano-team-b"]),
|
||||
(None, "gpt-5.4-nano", [_GLOBAL_BARE_NAME, _TEAM_B_BARE_COPY], ["id-nano"]),
|
||||
(None, "bedrock/us.amazon.nova-2-lite-v1:0", [_GLOBAL_DEPLOYMENT, _TEAM_B_COPY], ["id-bedrock", "id-team-b"]),
|
||||
],
|
||||
ids=[
|
||||
"a team-less caller reaches a public name nothing else carries",
|
||||
"model_name wins over a public name for a team-less caller",
|
||||
"a team's own copy wins over the global model_name",
|
||||
"a team's own copy wins over a litellm_params.model equal to the public name",
|
||||
"model_name wins over a litellm_params.model equal to it for a team-less caller",
|
||||
"a provider model string no name carries still matches litellm_params.model",
|
||||
],
|
||||
)
|
||||
async def test_perform_health_check_targets_a_name_the_way_a_request_for_it_routes(team_id, model_list, expected_ids):
|
||||
async def test_perform_health_check_targets_a_name_the_way_a_request_for_it_routes(
|
||||
team_id, model, model_list, expected_ids
|
||||
):
|
||||
"""``/health?model=<name>`` probes the deployments a request for that name from the same caller would route to."""
|
||||
from litellm.proxy.health_check import perform_health_check
|
||||
|
||||
|
|
@ -713,7 +731,7 @@ async def test_perform_health_check_targets_a_name_the_way_a_request_for_it_rout
|
|||
with patch( # test-quality-ok: the deployments handed to the probe are the assertion; no injection seam
|
||||
"litellm.proxy.health_check._perform_health_check", probe
|
||||
):
|
||||
healthy, unhealthy, _ = await perform_health_check(model_list=model_list, model="bedrock-nova", team_id=team_id)
|
||||
healthy, unhealthy, _ = await perform_health_check(model_list=model_list, model=model, team_id=team_id)
|
||||
|
||||
assert [m["model_info"]["id"] for m in probe.call_args.args[0]] == expected_ids
|
||||
assert [ep["model_id"] for ep in healthy] == expected_ids
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue