mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-11 08:38:24 +00:00
fix(vllm): list hosted_vllm wildcard models when the server has no api key (#40066)
* fix(proxy): expand wildcard models for providers with model info but no cost map entry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(vllm): stop requiring an api key to list vllm models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): preserve model ids during wildcard expansion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(vllm): forward api key for model discovery Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(vllm): support bearer auth for model discovery Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(vllm): list hosted_vllm models from the server chat uses, with a bounded wait The hosted_vllm listing now resolves HOSTED_VLLM_API_BASE and HOSTED_VLLM_API_KEY like hosted_vllm chat, instead of the /vllm pass-through's VLLM_API_BASE, and waits at most NON_LLM_CONNECTION_TIMEOUT rather than the completion timeout. vllm/* keeps listing nothing, since its chat runs in process. The router pass-through now forwards the deployment's api_key. * fix(vllm): skip the placeholder key, honor deployment TLS and headers, and retry failed listings after a minute * fix(discovery): cache only timed-out model listings, so a fast failure is tried again on the next call * fix(proxy): list wildcard models off the event loop, so a hung provider no longer stalls other requests * fix(utils): keep a concurrent call's model listing when another call times out * fix(proxy): run model discovery on its own thread pool and keep credential lookups off the event loop * fix(proxy): stop the file credential lookup at the first match before listing wildcard models The fallback lookup now walks the allowed model names lazily, in the same order as before, so a hung vLLM server is listed only when no configured name serves the requested provider. The vLLM auth headers no longer mutate the caller's headers dict. * fix(proxy): try every configured wildcard route before listing any wildcard's models in the file credential lookup * fix(proxy): answer model lists from cached listings first and share one listing per server * test(vllm): give the self-signed cert fixture fixed validity dates * fix(proxy): skip wildcard servers a key cannot reach on /v1/model/info /v1/model/info listed every wildcard deployment and filtered by the key afterwards, so a key limited to one model waited on every wildcard server and sent listing calls to servers it cannot use. The listing now skips a wildcard whose patterns match none of the key's models before it asks the server. A key or team granted hosted_vllm/* with no deployment of that name now lists no models, like openrouter/*, since chat answers 400 for every id the environment server would list. * fix(proxy): match a wildcard's own row pattern before asking its server The /v1/model/info pre-filter matched a key's models against both the deployment name and its litellm_params.model, so a key granted hosted_vllm/* still asked a vllm-secure/* -> hosted_vllm/* server whose rows it can never see. It now matches only the pattern expansion names the rows with, picked by one shared helper. * test(vllm): cover the vLLM listings and no-slash wildcard aliases the patch left unrun Adds tests for hosted_vllm get_models with and without a server, vllm/* never asking a configured server, and /model/info pre-filtering of wildcard aliases whose name has no slash. Drops a test comment that only repeated a method signature. --------- Co-authored-by: jesus <jesus@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
2a94bb3835
commit
08b951a814
9 changed files with 555 additions and 46 deletions
|
|
@ -2,10 +2,12 @@ from collections.abc import Mapping
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT, PLACEHOLDER_API_KEY
|
||||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.custom_httpx.http_handler import get_httpx_client
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.utils import add_path_to_api_base
|
||||
|
|
@ -37,17 +39,31 @@ class VLLMError(BaseLLMException):
|
|||
)
|
||||
|
||||
|
||||
_EXTRA_HEADERS: Final = TypeAdapter(Mapping[str, str])
|
||||
|
||||
|
||||
def _auth_headers(api_key: str | None) -> Mapping[str, str]:
|
||||
if api_key is None or api_key == PLACEHOLDER_API_KEY:
|
||||
return {}
|
||||
return {"Authorization": f"Bearer {api_key}"}
|
||||
|
||||
|
||||
def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None:
|
||||
value: Final = litellm_params.get(key)
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
class VLLMModelInfo(BaseLLMModelInfo):
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
headers: dict[str, str],
|
||||
model: str,
|
||||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, str]:
|
||||
upstream_key: Final = self.upstream_api_key(api_key, api_base)
|
||||
return {**headers, **self.upstream_auth_headers(upstream_key)} if upstream_key else headers
|
||||
|
||||
|
|
@ -58,7 +74,7 @@ class VLLMModelInfo(BaseLLMModelInfo):
|
|||
return {"Authorization": f"Bearer {api_key}", "x-api-key": api_key}
|
||||
|
||||
@staticmethod
|
||||
def get_api_base(api_base: str | None = None) -> str | None:
|
||||
def get_api_base(api_base: str | None = None) -> str:
|
||||
api_base = api_base or get_secret_str("VLLM_API_BASE")
|
||||
if api_base is None:
|
||||
raise ValueError(
|
||||
|
|
@ -74,25 +90,55 @@ class VLLMModelInfo(BaseLLMModelInfo):
|
|||
def get_base_model(model: str) -> str | None:
|
||||
return model
|
||||
|
||||
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
|
||||
api_base = VLLMModelInfo.get_api_base(api_base)
|
||||
api_key = VLLMModelInfo.get_api_key(api_key)
|
||||
endpoint: Final = "/v1/models"
|
||||
if api_base is None or api_key is None:
|
||||
raise ValueError(
|
||||
"VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint."
|
||||
)
|
||||
|
||||
url: Final = add_path_to_api_base(api_base, endpoint)
|
||||
response: Final = litellm.module_level_client.get(
|
||||
url=url,
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
models: Final = response.json()["data"]
|
||||
|
||||
return [model["id"] for model in models]
|
||||
def get_models(
|
||||
self, api_key: str | None = None, api_base: str | None = None
|
||||
) -> list[str]: # mutable-ok: mirrors override contract
|
||||
return []
|
||||
|
||||
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
|
||||
return VLLMError(status_code=status_code, message=error_message, headers=headers)
|
||||
|
||||
|
||||
class HostedVLLMModelInfo(VLLMModelInfo):
|
||||
@staticmethod
|
||||
def get_api_base(api_base: str | None = None) -> str:
|
||||
resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE")
|
||||
if resolved_api_base is None:
|
||||
raise ValueError("HOSTED_VLLM_API_BASE is not set.")
|
||||
return resolved_api_base
|
||||
|
||||
@staticmethod
|
||||
def get_api_key(api_key: str | None = None) -> str | None:
|
||||
return api_key or get_secret_str("HOSTED_VLLM_API_KEY")
|
||||
|
||||
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
|
||||
return self._list_models(api_key=api_key, api_base=api_base, extra_headers={}, ssl_verify=None)
|
||||
|
||||
def discover_models(
|
||||
self, litellm_params: Mapping[str, object] | None = None
|
||||
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
|
||||
params: Final = litellm_params or {}
|
||||
ssl_verify: Final = params.get("ssl_verify")
|
||||
return self._list_models(
|
||||
api_key=_str_param(params, "api_key"),
|
||||
api_base=_str_param(params, "api_base"),
|
||||
extra_headers=_EXTRA_HEADERS.validate_python(params.get("extra_headers") or {}),
|
||||
ssl_verify=ssl_verify if isinstance(ssl_verify, (bool, str)) else None,
|
||||
)
|
||||
|
||||
def _list_models(
|
||||
self,
|
||||
*,
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
extra_headers: Mapping[str, str],
|
||||
ssl_verify: bool | str | None,
|
||||
) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override
|
||||
url: Final = add_path_to_api_base(self.get_api_base(api_base), "/v1/models")
|
||||
response: Final = get_httpx_client(params={"ssl_verify": ssl_verify}).get(
|
||||
url=url,
|
||||
headers={**extra_headers, **_auth_headers(self.get_api_key(api_key))},
|
||||
timeout=NON_LLM_CONNECTION_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return [model["id"] for model in response.json()["data"]]
|
||||
|
|
|
|||
|
|
@ -67,9 +67,6 @@ class VLLMPassthroughConfig(VLLMModelInfo, BasePassthroughConfig):
|
|||
) -> tuple["URL", str]:
|
||||
base_target_url: Final = self.get_api_base(api_base)
|
||||
|
||||
if base_target_url is None:
|
||||
raise Exception("VLLM api base not found")
|
||||
|
||||
return (
|
||||
self.format_url(endpoint_under_api_base(endpoint, base_target_url), base_target_url, request_query_params),
|
||||
base_target_url,
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import copy
|
|||
import functools
|
||||
import itertools
|
||||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from collections.abc import Callable, Collection, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeVar
|
||||
|
|
@ -35,7 +35,9 @@ _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
|
|||
|
||||
_MODEL_LISTING_POOL: Final = ThreadPoolExecutor(thread_name_prefix="litellm-model-listing")
|
||||
|
||||
_PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT: Final = frozenset({LlmProviders.OPENROUTER.value})
|
||||
_PROVIDERS_LISTED_ONLY_THROUGH_A_DEPLOYMENT: Final = frozenset(
|
||||
{LlmProviders.OPENROUTER.value, LlmProviders.HOSTED_VLLM.value}
|
||||
)
|
||||
|
||||
_Listed: Final = TypeVar("_Listed")
|
||||
|
||||
|
|
@ -562,9 +564,27 @@ def _names_no_deployment_serves_by_name(
|
|||
return tuple(name for name in names if not deployed_by_name(name, team_id))
|
||||
|
||||
|
||||
def _wildcard_pattern_to_expand(model_name: str, litellm_model: str) -> str | None:
|
||||
if not _check_wildcard_routing(model_name):
|
||||
return None
|
||||
if "/" in model_name or model_name == "*":
|
||||
return model_name
|
||||
if _check_wildcard_routing(litellm_model):
|
||||
return litellm_model
|
||||
return model_name
|
||||
|
||||
|
||||
def _can_expand_to_an_allowed_name(wildcard_pattern: str, allowed_model_names: Collection[str] | None) -> bool:
|
||||
if allowed_model_names is None:
|
||||
return True
|
||||
pattern: Final = _wildcard_name_regex(wildcard_pattern)
|
||||
return any(pattern.match(name) for name in allowed_model_names)
|
||||
|
||||
|
||||
def expand_wildcard_deployments_for_model_info(
|
||||
deployments: list[dict[str, object]],
|
||||
llm_router: Router | None = None,
|
||||
allowed_model_names: Collection[str] | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Expand wildcard deployments into one row per known provider model.
|
||||
|
||||
|
|
@ -581,19 +601,11 @@ def expand_wildcard_deployments_for_model_info(
|
|||
raw_params = deployment.get("litellm_params")
|
||||
litellm_params_dict: dict[str, object] = raw_params if isinstance(raw_params, dict) else {}
|
||||
litellm_model = str(litellm_params_dict.get("model") or "")
|
||||
|
||||
# Determine the wildcard pattern to expand.
|
||||
# Branch order matters: only fall to litellm_model when model_name is
|
||||
# also a wildcard, so a concrete model_name is never overwritten.
|
||||
if _check_wildcard_routing(model_name) and ("/" in model_name or model_name == "*"):
|
||||
wildcard_pattern = model_name
|
||||
elif _check_wildcard_routing(model_name) and _check_wildcard_routing(litellm_model):
|
||||
wildcard_pattern = litellm_model
|
||||
elif _check_wildcard_routing(model_name):
|
||||
wildcard_pattern = model_name
|
||||
else:
|
||||
if (wildcard_pattern := _wildcard_pattern_to_expand(model_name, litellm_model)) is None:
|
||||
expanded.append(deployment)
|
||||
continue
|
||||
if not _can_expand_to_an_allowed_name(wildcard_pattern, allowed_model_names):
|
||||
continue
|
||||
|
||||
try:
|
||||
litellm_params = LiteLLM_Params.model_validate(litellm_params_dict) if litellm_params_dict else None
|
||||
|
|
|
|||
|
|
@ -18409,8 +18409,6 @@ async def model_info_v1(
|
|||
alias_models: Final = copy.deepcopy(registry.deployments(source="alias"))
|
||||
all_models.extend(alias_models)
|
||||
|
||||
all_models = await run_model_listing(partial(expand_wildcard_deployments_for_model_info, all_models, llm_router))
|
||||
|
||||
allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
llm_router=llm_router,
|
||||
|
|
@ -18419,6 +18417,15 @@ async def model_info_v1(
|
|||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
|
||||
all_models = await run_model_listing(
|
||||
partial(
|
||||
expand_wildcard_deployments_for_model_info,
|
||||
all_models,
|
||||
llm_router,
|
||||
allowed_model_names=allowed_model_names,
|
||||
)
|
||||
)
|
||||
|
||||
all_models = _filter_v1_model_info_deployments(
|
||||
all_models=all_models,
|
||||
allowed_model_names=allowed_model_names,
|
||||
|
|
|
|||
|
|
@ -10558,12 +10558,14 @@ class ProviderConfigManager:
|
|||
from litellm.llms.ollama.common_utils import OllamaModelInfo
|
||||
|
||||
return OllamaModelInfo()
|
||||
elif LlmProviders.VLLM == provider or LlmProviders.HOSTED_VLLM == provider:
|
||||
from litellm.llms.vllm.common_utils import (
|
||||
VLLMModelInfo, # experimental approach, to reduce bloat on __init__.py
|
||||
)
|
||||
elif LlmProviders.VLLM == provider:
|
||||
from litellm.llms.vllm.common_utils import VLLMModelInfo
|
||||
|
||||
return VLLMModelInfo()
|
||||
elif LlmProviders.HOSTED_VLLM == provider:
|
||||
from litellm.llms.vllm.common_utils import HostedVLLMModelInfo
|
||||
|
||||
return HostedVLLMModelInfo()
|
||||
elif LlmProviders.LEMONADE == provider:
|
||||
return litellm.LemonadeChatConfig()
|
||||
elif LlmProviders.CLARIFAI == provider:
|
||||
|
|
|
|||
27
tests/unit/llms/vllm/test_vllm_common_utils.py
Normal file
27
tests/unit/llms/vllm/test_vllm_common_utils.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from litellm.llms.vllm.common_utils import HostedVLLMModelInfo
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_get_models_lists_the_server_ids_sending_the_key_as_a_bearer_token() -> None:
|
||||
respx.get("http://vllm-sdk.test:8000/v1/models", headers={"Authorization": "Bearer vllm-sdk-key"}).mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}, {"id": "meta-llama/Llama-3.1-8B-Instruct"}]})
|
||||
)
|
||||
|
||||
listed: Final = HostedVLLMModelInfo().get_models(api_key="vllm-sdk-key", api_base="http://vllm-sdk.test:8000/v1")
|
||||
|
||||
assert listed == ["qwen2.5", "meta-llama/Llama-3.1-8B-Instruct"]
|
||||
|
||||
|
||||
def test_hosted_vllm_get_models_without_a_server_names_the_setting_that_is_missing(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
|
||||
|
||||
with pytest.raises(ValueError, match="HOSTED_VLLM_API_BASE"):
|
||||
HostedVLLMModelInfo().get_models()
|
||||
|
|
@ -1,11 +1,22 @@
|
|||
from collections.abc import Generator, Mapping
|
||||
import datetime
|
||||
import ipaddress
|
||||
import json
|
||||
import ssl
|
||||
import threading
|
||||
from collections.abc import Generator, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import rsa
|
||||
from cryptography.x509.oid import NameOID
|
||||
|
||||
from litellm import Router
|
||||
|
||||
|
|
@ -430,6 +441,262 @@ def test_wildcard_credential_hydration_preserves_deployment_params(
|
|||
}
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_lists_the_server_models_without_an_api_key(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
models_route: Final = respx.get("http://vllm-without-key.test:8000/v1/models").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"data": [
|
||||
{"id": "meta-llama/Llama-3.1-8B-Instruct"},
|
||||
{"id": "qwen2.5"},
|
||||
{"id": "openai/gpt-oss-20b"},
|
||||
]
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*",
|
||||
LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-without-key.test:8000/v1"),
|
||||
)
|
||||
|
||||
assert listed == [
|
||||
"hosted_vllm/meta-llama/Llama-3.1-8B-Instruct",
|
||||
"hosted_vllm/qwen2.5",
|
||||
"hosted_vllm/openai/gpt-oss-20b",
|
||||
]
|
||||
assert "authorization" not in models_route.calls.last.request.headers
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_sends_the_configured_api_key_as_a_bearer_token(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
models_route: Final = respx.get(
|
||||
"http://vllm-with-key.test:8000/v1/models", headers={"Authorization": "Bearer vllm-test-key"}
|
||||
).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*",
|
||||
LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-with-key.test:8000/v1", api_key="vllm-test-key"),
|
||||
)
|
||||
|
||||
assert listed == ["hosted_vllm/qwen2.5"]
|
||||
assert models_route.call_count == 1
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_without_api_base_lists_the_server_chat_would_call(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm-env.test:8000/v1")
|
||||
monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-env-key")
|
||||
monkeypatch.setenv("VLLM_API_BASE", "http://vllm-passthrough-env.test:8000")
|
||||
passthrough_server_route: Final = respx.get("http://vllm-passthrough-env.test:8000/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "model-on-the-passthrough-server"}]})
|
||||
)
|
||||
respx.get(
|
||||
"http://hosted-vllm-env.test:8000/v1/models", headers={"Authorization": "Bearer hosted-vllm-env-key"}
|
||||
).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", custom_llm_provider="hosted_vllm")
|
||||
)
|
||||
|
||||
assert listed == ["hosted_vllm/qwen2.5"]
|
||||
assert not passthrough_server_route.called
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_hung_server(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
models_route: Final = respx.get("http://vllm-timeout.test:8000/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
)
|
||||
|
||||
get_known_models_from_wildcard(
|
||||
"hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-timeout.test:8000/v1")
|
||||
)
|
||||
|
||||
read_timeout: Final = models_route.calls.last.request.extensions["timeout"]["read"]
|
||||
assert read_timeout is not None
|
||||
assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
respx.get(
|
||||
"http://vllm-behind-gateway.test:8000/v1/models",
|
||||
headers={"X-Tenant": "team-a", "Authorization": "Bearer vllm-test-key"},
|
||||
).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*",
|
||||
LiteLLM_Params(
|
||||
model="hosted_vllm/*",
|
||||
api_base="http://vllm-behind-gateway.test:8000/v1",
|
||||
api_key="vllm-test-key",
|
||||
extra_headers={"X-Tenant": "team-a"},
|
||||
),
|
||||
)
|
||||
|
||||
assert listed == ["hosted_vllm/qwen2.5"]
|
||||
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_vllm_wildcard_never_lists_the_ids_of_a_configured_vllm_server(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.setenv("VLLM_API_BASE", "http://vllm-library-env.test:8000")
|
||||
monkeypatch.setenv("VLLM_API_KEY", "vllm-library-env-key")
|
||||
server_route: Final = respx.get(url__regex=r"http://vllm-library(-env)?\.test:8000/.*").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "model-only-a-server-serves"}]})
|
||||
)
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"vllm/*",
|
||||
LiteLLM_Params(model="vllm/*", api_base="http://vllm-library.test:8000", api_key="vllm-library-key"),
|
||||
)
|
||||
|
||||
assert "vllm/model-only-a-server-serves" not in listed
|
||||
assert not server_route.called
|
||||
|
||||
_SELF_SIGNED_CERT_NOT_VALID_BEFORE: Final = datetime.datetime(2000, 1, 1, tzinfo=datetime.UTC)
|
||||
_SELF_SIGNED_CERT_NOT_VALID_AFTER: Final = datetime.datetime(9999, 12, 31, tzinfo=datetime.UTC)
|
||||
|
||||
|
||||
def _self_signed_cert(directory: Path) -> tuple[Path, Path]:
|
||||
key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
|
||||
name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")])
|
||||
cert: Final = (
|
||||
x509.CertificateBuilder()
|
||||
.subject_name(name)
|
||||
.issuer_name(name)
|
||||
.public_key(key.public_key())
|
||||
.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(_SELF_SIGNED_CERT_NOT_VALID_BEFORE)
|
||||
.not_valid_after(_SELF_SIGNED_CERT_NOT_VALID_AFTER)
|
||||
.add_extension(
|
||||
x509.SubjectAlternativeName([x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))]), critical=False
|
||||
)
|
||||
.sign(key, hashes.SHA256())
|
||||
)
|
||||
cert_path: Final = directory / "cert.pem"
|
||||
key_path: Final = directory / "key.pem"
|
||||
cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM))
|
||||
key_path.write_bytes(
|
||||
key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
|
||||
)
|
||||
return cert_path, key_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def self_signed_vllm_api_base(tmp_path: Path) -> Iterator[str]:
|
||||
cert_path, key_path = _self_signed_cert(tmp_path)
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
timeout = 5
|
||||
|
||||
def do_GET(self) -> None:
|
||||
body: Final = json.dumps({"data": [{"id": "qwen2.5"}]}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
pass
|
||||
|
||||
server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.load_cert_chain(cert_path, key_path)
|
||||
server.socket = context.wrap_socket(server.socket, server_side=True, do_handshake_on_connect=False)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
try:
|
||||
yield f"https://127.0.0.1:{server.server_address[1]}/v1"
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def test_hosted_vllm_wildcard_lists_a_self_signed_server_only_when_the_deployment_turns_off_ssl_verify(
|
||||
monkeypatch: pytest.MonkeyPatch, self_signed_vllm_api_base: str
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
from litellm.llms.custom_httpx import http_handler
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.setattr(litellm, "ssl_verify", True)
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
monkeypatch.setattr(http_handler, "_ssl_context_cache", {})
|
||||
monkeypatch.delenv("SSL_VERIFY", raising=False)
|
||||
monkeypatch.delenv("SSL_CERT_FILE", raising=False)
|
||||
|
||||
verified: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base)
|
||||
)
|
||||
unverified: Final = get_known_models_from_wildcard(
|
||||
"hosted_vllm/*",
|
||||
LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base, ssl_verify=False),
|
||||
)
|
||||
|
||||
assert verified == []
|
||||
assert unverified == ["hosted_vllm/qwen2.5"]
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_custom_prefix_over_a_litellm_proxy_wildcard_lists_upstream_models_under_the_prefix_alone(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
respx.get("http://upstream-proxy.test:4000/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "gpt-4o"}, {"id": "claude-sonnet"}]})
|
||||
)
|
||||
|
||||
listed: Final = get_known_models_from_wildcard(
|
||||
"team-proxy/*",
|
||||
LiteLLM_Params(model="litellm_proxy/*", api_base="http://upstream-proxy.test:4000", api_key="sk-upstream"),
|
||||
)
|
||||
|
||||
assert listed == ["team-proxy/gpt-4o", "team-proxy/claude-sonnet"]
|
||||
|
||||
|
||||
def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch):
|
||||
"""Regression test for #30358.
|
||||
|
||||
|
|
@ -834,6 +1101,56 @@ def test_expand_wildcard_invalid_litellm_params_passthrough():
|
|||
assert result == [deployment]
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("allowed_model_names", "expected_names", "expected_listings"),
|
||||
[
|
||||
pytest.param(None, ["hosted_vllm/qwen2.5"], 1, id="unscoped"),
|
||||
pytest.param(["hosted_vllm/qwen2.5"], ["hosted_vllm/qwen2.5"], 1, id="granted-a-listed-id"),
|
||||
pytest.param(["qwen-open"], [], 0, id="granted-a-name-the-alias-matches"),
|
||||
],
|
||||
)
|
||||
@respx.mock
|
||||
def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_provider_pattern(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
request: pytest.FixtureRequest,
|
||||
allowed_model_names: list[str] | None,
|
||||
expected_names: list[str],
|
||||
expected_listings: int,
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
api_base: Final = f"http://vllm-alias-{request.node.callspec.id}.test:8000/v1"
|
||||
listing: Final = respx.get(f"{api_base}/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
)
|
||||
deployment: Final = {"model_name": "qwen-*", "litellm_params": {"model": "hosted_vllm/*", "api_base": api_base}}
|
||||
|
||||
rows: Final = expand_wildcard_deployments_for_model_info([deployment], allowed_model_names=allowed_model_names)
|
||||
|
||||
assert ([row["model_name"] for row in rows], listing.call_count) == (expected_names, expected_listings)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("allowed_model_names", "keeps_the_alias"),
|
||||
[
|
||||
pytest.param(["gpt-4o-mini"], True, id="granted-a-name-the-alias-matches"),
|
||||
pytest.param(["claude-sonnet"], False, id="granted-only-other-names"),
|
||||
],
|
||||
)
|
||||
def test_expand_wildcard_alias_over_one_model_is_kept_only_for_keys_with_a_name_it_matches(
|
||||
allowed_model_names: list[str], keeps_the_alias: bool
|
||||
) -> None:
|
||||
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info
|
||||
|
||||
deployment: Final = {"model_name": "gpt-4o*", "litellm_params": {"model": "openai/gpt-4o"}}
|
||||
|
||||
rows: Final = expand_wildcard_deployments_for_model_info([deployment], allowed_model_names=allowed_model_names)
|
||||
|
||||
assert rows == ([deployment] if keeps_the_alias else [])
|
||||
|
||||
def test_get_complete_model_list_excludes_wildcard_routes_by_default():
|
||||
"""Regression (LIT-4108): a wildcard with a matching router deployment leaked into /v1/models."""
|
||||
from litellm import Router
|
||||
|
|
@ -1327,6 +1644,40 @@ def test_key_granted_openrouter_wildcard_without_an_openrouter_deployment_lists_
|
|||
assert (listed, catalog.call_count) == ([], 0)
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_key_granted_hosted_vllm_wildcard_without_a_hosted_vllm_deployment_lists_no_environment_server(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm-environment-only.test:8000/v1")
|
||||
environment_server: Final = respx.get("http://hosted-vllm-environment-only.test:8000/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "vllm-team/*",
|
||||
"litellm_params": {"model": "hosted_vllm/*", "api_base": "http://vllm-team.test:8000/v1"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
listed: Final = get_complete_model_list(
|
||||
key_models=["hosted_vllm/*"],
|
||||
team_models=[],
|
||||
proxy_model_list=router.model_names,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
llm_router=router,
|
||||
)
|
||||
|
||||
assert (listed, environment_server.call_count) == ([], 0)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _openai_catalog_behind_a_catch_all(
|
||||
monkeypatch: pytest.MonkeyPatch, concrete_name: str, concrete_model_info: Mapping[str, str]
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from unittest.mock import AsyncMock, MagicMock
|
|||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import litellm
|
||||
|
|
@ -576,6 +577,55 @@ async def test_v1_model_info_star_wildcard_filter_keeps_provider_expansion(monke
|
|||
assert router.registry().names("gpt-4o", kind="wildcard") == frozenset({"openai/gpt-4o"})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("key_models", "expected_names", "expected_listings"),
|
||||
[(["qwen-open"], ["qwen-open"], 0), (["hosted_vllm/*"], ["hosted_vllm/qwen2.5"], 1)],
|
||||
ids=["key-without-the-wildcard", "key-granted-the-wildcard"],
|
||||
)
|
||||
def test_v1_model_info_asks_a_wildcard_server_only_for_keys_that_can_call_it(
|
||||
client, auth_as, monkeypatch, request, key_models, expected_names, expected_listings
|
||||
):
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
wildcard_server: Final = f"http://vllm-{request.node.callspec.id}.test:8000/v1"
|
||||
other_alias_server: Final = f"http://vllm-secure-{request.node.callspec.id}.test:8000/v1"
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "qwen-open",
|
||||
"litellm_params": {"model": "hosted_vllm/qwen2.5", "api_base": "http://vllm-open.test:8000/v1"},
|
||||
},
|
||||
{"model_name": "hosted_vllm/*", "litellm_params": {"model": "hosted_vllm/*", "api_base": wildcard_server}},
|
||||
{
|
||||
"model_name": "vllm-secure/*",
|
||||
"litellm_params": {"model": "hosted_vllm/*", "api_base": other_alias_server},
|
||||
},
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "llm_model_list", router.get_model_list())
|
||||
monkeypatch.setattr(proxy_server, "user_model", None)
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", None)
|
||||
|
||||
with respx.mock(assert_all_called=False) as upstream:
|
||||
listing: Final = upstream.get(f"{wildcard_server}/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
)
|
||||
other_alias_listing: Final = upstream.get(f"{other_alias_server}/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
)
|
||||
with auth_as(role=LitellmUserRoles.INTERNAL_USER, models=key_models):
|
||||
response: Final = client.get("/v1/model/info")
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert (
|
||||
[model["model_name"] for model in response.json()["data"]],
|
||||
listing.call_count,
|
||||
other_alias_listing.call_count,
|
||||
) == (expected_names, expected_listings, 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GET /model/info — team BYOK scoping (issue #30983)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -9350,6 +9350,23 @@ def test_get_valid_models_does_not_ask_a_timed_out_provider_again_on_the_next_li
|
|||
assert (listings, catalog.call_count) == (([], []), 1)
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_get_valid_models_does_not_ask_a_timed_out_hosted_vllm_server_again_on_the_next_listing(
|
||||
reset_mock_cache: None,
|
||||
) -> None:
|
||||
models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock(
|
||||
side_effect=httpx.ReadTimeout("vLLM did not answer")
|
||||
)
|
||||
deployment: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-hung.test:8000/v1")
|
||||
|
||||
listings: Final = tuple(
|
||||
get_valid_models(check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=deployment)
|
||||
for _ in range(2)
|
||||
)
|
||||
|
||||
assert (listings, models_route.call_count) == (([], []), 1)
|
||||
|
||||
|
||||
@respx.mock
|
||||
def test_get_valid_models_lists_a_provider_again_right_after_a_fast_failure(reset_mock_cache: None) -> None:
|
||||
respx.get("https://openrouter.ai/api/v1/models/user").mock(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue