mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-09 08:18:12 +00:00
fix(proxy): match a wildcard's own row pattern before asking its server
The /v1/model/info pre-filter matched a key's models against both the deployment name and its litellm_params.model, so a key granted hosted_vllm/* still asked a vllm-secure/* -> hosted_vllm/* server whose rows it can never see. It now matches only the pattern expansion names the rows with, picked by one shared helper.
This commit is contained in:
parent
b7c443b440
commit
fef95dc90e
2 changed files with 28 additions and 25 deletions
|
|
@ -564,17 +564,21 @@ def _names_no_deployment_serves_by_name(
|
|||
return tuple(name for name in names if not deployed_by_name(name, team_id))
|
||||
|
||||
|
||||
def _matches_a_pattern(name: str, patterns: Sequence[str]) -> bool:
|
||||
return any(_wildcard_name_regex(pattern).match(name) for pattern in patterns)
|
||||
def _wildcard_pattern_to_expand(model_name: str, litellm_model: str) -> str | None:
|
||||
if not _check_wildcard_routing(model_name):
|
||||
return None
|
||||
if "/" in model_name or model_name == "*":
|
||||
return model_name
|
||||
if _check_wildcard_routing(litellm_model):
|
||||
return litellm_model
|
||||
return model_name
|
||||
|
||||
|
||||
def _can_expand_to_an_allowed_name(
|
||||
model_name: str, litellm_model: str, allowed_model_names: Collection[str] | None
|
||||
) -> bool:
|
||||
if allowed_model_names is None or not _check_wildcard_routing(model_name):
|
||||
def _can_expand_to_an_allowed_name(wildcard_pattern: str, allowed_model_names: Collection[str] | None) -> bool:
|
||||
if allowed_model_names is None:
|
||||
return True
|
||||
patterns: Final = tuple(pattern for pattern in (model_name, litellm_model) if _check_wildcard_routing(pattern))
|
||||
return any(_matches_a_pattern(name, patterns) for name in allowed_model_names)
|
||||
pattern: Final = _wildcard_name_regex(wildcard_pattern)
|
||||
return any(pattern.match(name) for name in allowed_model_names)
|
||||
|
||||
|
||||
def expand_wildcard_deployments_for_model_info(
|
||||
|
|
@ -597,21 +601,11 @@ def expand_wildcard_deployments_for_model_info(
|
|||
raw_params = deployment.get("litellm_params")
|
||||
litellm_params_dict: dict[str, object] = raw_params if isinstance(raw_params, dict) else {}
|
||||
litellm_model = str(litellm_params_dict.get("model") or "")
|
||||
if not _can_expand_to_an_allowed_name(model_name, litellm_model, allowed_model_names):
|
||||
continue
|
||||
|
||||
# Determine the wildcard pattern to expand.
|
||||
# Branch order matters: only fall to litellm_model when model_name is
|
||||
# also a wildcard, so a concrete model_name is never overwritten.
|
||||
if _check_wildcard_routing(model_name) and ("/" in model_name or model_name == "*"):
|
||||
wildcard_pattern = model_name
|
||||
elif _check_wildcard_routing(model_name) and _check_wildcard_routing(litellm_model):
|
||||
wildcard_pattern = litellm_model
|
||||
elif _check_wildcard_routing(model_name):
|
||||
wildcard_pattern = model_name
|
||||
else:
|
||||
if (wildcard_pattern := _wildcard_pattern_to_expand(model_name, litellm_model)) is None:
|
||||
expanded.append(deployment)
|
||||
continue
|
||||
if not _can_expand_to_an_allowed_name(wildcard_pattern, allowed_model_names):
|
||||
continue
|
||||
|
||||
try:
|
||||
litellm_params = LiteLLM_Params.model_validate(litellm_params_dict) if litellm_params_dict else None
|
||||
|
|
|
|||
|
|
@ -588,6 +588,7 @@ def test_v1_model_info_asks_a_wildcard_server_only_for_keys_that_can_call_it(
|
|||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
wildcard_server: Final = f"http://vllm-{request.node.callspec.id}.test:8000/v1"
|
||||
other_alias_server: Final = f"http://vllm-secure-{request.node.callspec.id}.test:8000/v1"
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
|
|
@ -596,6 +597,10 @@ def test_v1_model_info_asks_a_wildcard_server_only_for_keys_that_can_call_it(
|
|||
"litellm_params": {"model": "hosted_vllm/qwen2.5", "api_base": "http://vllm-open.test:8000/v1"},
|
||||
},
|
||||
{"model_name": "hosted_vllm/*", "litellm_params": {"model": "hosted_vllm/*", "api_base": wildcard_server}},
|
||||
{
|
||||
"model_name": "vllm-secure/*",
|
||||
"litellm_params": {"model": "hosted_vllm/*", "api_base": other_alias_server},
|
||||
},
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
|
|
@ -607,14 +612,18 @@ def test_v1_model_info_asks_a_wildcard_server_only_for_keys_that_can_call_it(
|
|||
listing: Final = upstream.get(f"{wildcard_server}/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
)
|
||||
other_alias_listing: Final = upstream.get(f"{other_alias_server}/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
)
|
||||
with auth_as(role=LitellmUserRoles.INTERNAL_USER, models=key_models):
|
||||
response: Final = client.get("/v1/model/info")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert ([model["model_name"] for model in response.json()["data"]], listing.call_count) == (
|
||||
expected_names,
|
||||
expected_listings,
|
||||
)
|
||||
assert (
|
||||
[model["model_name"] for model in response.json()["data"]],
|
||||
listing.call_count,
|
||||
other_alias_listing.call_count,
|
||||
) == (expected_names, expected_listings, 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue