mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 9eae781a7a into 4ce724e3df
This commit is contained in:
commit
dddecb3a7d
2 changed files with 71 additions and 10 deletions
|
|
@ -1,3 +1,4 @@
|
|||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -45,33 +46,39 @@ class VLLMModelInfo(BaseLLMModelInfo):
|
|||
|
||||
@staticmethod
|
||||
def get_api_base(api_base: str | None = None) -> str | None:
|
||||
api_base = api_base or get_secret_str("VLLM_API_BASE")
|
||||
if api_base is None:
|
||||
resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE") or get_secret_str("VLLM_API_BASE")
|
||||
if resolved_api_base is None:
|
||||
raise ValueError(
|
||||
"VLLM_API_BASE is not set. Please set the environment variable, to use VLLM's pass-through - `{LITELLM_API_BASE}/vllm/{endpoint}`."
|
||||
"HOSTED_VLLM_API_BASE or VLLM_API_BASE is not set. Please set an environment variable to use "
|
||||
"VLLM's pass-through - `{LITELLM_API_BASE}/vllm/{endpoint}`."
|
||||
)
|
||||
return api_base
|
||||
return resolved_api_base
|
||||
|
||||
@staticmethod
|
||||
def get_api_key(api_key: str | None = None) -> str | None:
|
||||
return None
|
||||
return api_key or get_secret_str("HOSTED_VLLM_API_KEY")
|
||||
|
||||
@staticmethod
|
||||
def get_base_model(model: str) -> str | None:
|
||||
return model
|
||||
|
||||
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
|
||||
api_base = VLLMModelInfo.get_api_base(api_base)
|
||||
api_key = VLLMModelInfo.get_api_key(api_key)
|
||||
resolved_api_base: Final = VLLMModelInfo.get_api_base(api_base)
|
||||
resolved_api_key: Final = VLLMModelInfo.get_api_key(api_key)
|
||||
endpoint: Final = "/v1/models"
|
||||
if api_base is None or api_key is None:
|
||||
if resolved_api_base is None:
|
||||
raise ValueError(
|
||||
"VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint."
|
||||
"HOSTED_VLLM_API_BASE or VLLM_API_BASE is not set. Please set an environment variable to query "
|
||||
"VLLM's `/models` endpoint."
|
||||
)
|
||||
|
||||
url: Final = add_path_to_api_base(api_base, endpoint)
|
||||
url: Final = add_path_to_api_base(resolved_api_base, endpoint)
|
||||
headers: Final = (
|
||||
MappingProxyType({"Authorization": f"Bearer {resolved_api_key}"}) if resolved_api_key is not None else None
|
||||
)
|
||||
response: Final = litellm.module_level_client.get(
|
||||
url=url,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
|
|
|
|||
54
tests/unit/llms/vllm/test_common_utils.py
Normal file
54
tests/unit/llms/vllm/test_common_utils.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from litellm.llms.vllm.common_utils import VLLMModelInfo
|
||||
|
||||
MODELS_URL: Final = "http://vllm.example/v1/models"
|
||||
MODEL_RESPONSE: Final = {"data": [{"id": "Qwen/Qwen3-8B"}]}
|
||||
|
||||
|
||||
def test_get_models_without_api_key_sends_no_authorization_header(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
|
||||
|
||||
with respx.mock:
|
||||
route: Final = respx.get(MODELS_URL).mock(return_value=httpx.Response(200, json=MODEL_RESPONSE))
|
||||
models: Final = VLLMModelInfo().get_models(api_base="http://vllm.example/v1")
|
||||
|
||||
assert models == ["Qwen/Qwen3-8B"]
|
||||
assert "authorization" not in route.calls[0].request.headers
|
||||
|
||||
|
||||
def test_get_models_sends_api_key_as_bearer_token() -> None:
|
||||
with respx.mock:
|
||||
route: Final = respx.get(MODELS_URL).mock(return_value=httpx.Response(200, json=MODEL_RESPONSE))
|
||||
models: Final = VLLMModelInfo().get_models(
|
||||
api_key="vllm-test-key",
|
||||
api_base="http://vllm.example/v1",
|
||||
)
|
||||
|
||||
assert models == ["Qwen/Qwen3-8B"]
|
||||
assert route.calls[0].request.headers["authorization"] == "Bearer vllm-test-key"
|
||||
|
||||
|
||||
def test_get_models_requires_api_base(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("HOSTED_VLLM_API_BASE", raising=False)
|
||||
monkeypatch.delenv("VLLM_API_BASE", raising=False)
|
||||
|
||||
with pytest.raises(ValueError, match="HOSTED_VLLM_API_BASE"):
|
||||
VLLMModelInfo().get_models(api_key="vllm-test-key")
|
||||
|
||||
|
||||
def test_get_api_key_falls_back_to_hosted_vllm_environment_variable(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-test-key")
|
||||
|
||||
assert VLLMModelInfo.get_api_key() == "hosted-vllm-test-key"
|
||||
|
||||
|
||||
def test_get_api_base_prefers_hosted_vllm_environment_variable(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm.example/v1")
|
||||
monkeypatch.setenv("VLLM_API_BASE", "http://vllm.example/v1")
|
||||
|
||||
assert VLLMModelInfo.get_api_base() == "http://hosted-vllm.example/v1"
|
||||
Loading…
Add table
Reference in a new issue