This commit is contained in:
Mateo Wang 2026-10-29 13:25:29 -07:00 • committed by GitHub
commit dddecb3a7d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 71 additions and 10 deletions

View file

@ -1,3 +1,4 @@
from types import MappingProxyType
from typing import Final
import httpx
@ -45,33 +46,39 @@ class VLLMModelInfo(BaseLLMModelInfo):
@staticmethod
def get_api_base(api_base: str | None = None) -> str | None:
api_base = api_base or get_secret_str("VLLM_API_BASE")
if api_base is None:
resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE") or get_secret_str("VLLM_API_BASE")
if resolved_api_base is None:
raise ValueError(
"VLLM_API_BASE is not set. Please set the environment variable, to use VLLM's pass-through - `{LITELLM_API_BASE}/vllm/{endpoint}`."
"HOSTED_VLLM_API_BASE or VLLM_API_BASE is not set. Please set an environment variable to use "
"VLLM's pass-through - `{LITELLM_API_BASE}/vllm/{endpoint}`."
)
return api_base
return resolved_api_base
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return None
return api_key or get_secret_str("HOSTED_VLLM_API_KEY")
@staticmethod
def get_base_model(model: str) -> str | None:
return model
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
api_base = VLLMModelInfo.get_api_base(api_base)
api_key = VLLMModelInfo.get_api_key(api_key)
resolved_api_base: Final = VLLMModelInfo.get_api_base(api_base)
resolved_api_key: Final = VLLMModelInfo.get_api_key(api_key)
endpoint: Final = "/v1/models"
if api_base is None or api_key is None:
if resolved_api_base is None:
raise ValueError(
"VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint."
"HOSTED_VLLM_API_BASE or VLLM_API_BASE is not set. Please set an environment variable to query "
"VLLM's `/models` endpoint."
)
url: Final = add_path_to_api_base(api_base, endpoint)
url: Final = add_path_to_api_base(resolved_api_base, endpoint)
headers: Final = (
MappingProxyType({"Authorization": f"Bearer {resolved_api_key}"}) if resolved_api_key is not None else None
)
response: Final = litellm.module_level_client.get(
url=url,
headers=headers,
)
response.raise_for_status()

View file

@ -0,0 +1,54 @@
from typing import Final
import httpx
import pytest
import respx
from litellm.llms.vllm.common_utils import VLLMModelInfo
MODELS_URL: Final = "http://vllm.example/v1/models"
MODEL_RESPONSE: Final = {"data": [{"id": "Qwen/Qwen3-8B"}]}
def test_get_models_without_api_key_sends_no_authorization_header(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
with respx.mock:
route: Final = respx.get(MODELS_URL).mock(return_value=httpx.Response(200, json=MODEL_RESPONSE))
models: Final = VLLMModelInfo().get_models(api_base="http://vllm.example/v1")
assert models == ["Qwen/Qwen3-8B"]
assert "authorization" not in route.calls[0].request.headers
def test_get_models_sends_api_key_as_bearer_token() -> None:
with respx.mock:
route: Final = respx.get(MODELS_URL).mock(return_value=httpx.Response(200, json=MODEL_RESPONSE))
models: Final = VLLMModelInfo().get_models(
api_key="vllm-test-key",
api_base="http://vllm.example/v1",
)
assert models == ["Qwen/Qwen3-8B"]
assert route.calls[0].request.headers["authorization"] == "Bearer vllm-test-key"
def test_get_models_requires_api_base(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
monkeypatch.delenv("VLLM_API_BASE", raising=False)
with pytest.raises(ValueError, match="HOSTED_VLLM_API_BASE"):
VLLMModelInfo().get_models(api_key="vllm-test-key")
def test_get_api_key_falls_back_to_hosted_vllm_environment_variable(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-test-key")
assert VLLMModelInfo.get_api_key() == "hosted-vllm-test-key"
def test_get_api_base_prefers_hosted_vllm_environment_variable(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm.example/v1")
monkeypatch.setenv("VLLM_API_BASE", "http://vllm.example/v1")
assert VLLMModelInfo.get_api_base() == "http://hosted-vllm.example/v1"