fix(vllm): list hosted_vllm models from the server chat uses, with a bounded wait

The hosted_vllm listing now resolves HOSTED_VLLM_API_BASE and HOSTED_VLLM_API_KEY like hosted_vllm chat, instead of the /vllm pass-through's VLLM_API_BASE, and waits at most NON_LLM_CONNECTION_TIMEOUT rather than the completion timeout. vllm/* keeps listing nothing, since its chat runs in process. The router pass-through now forwards the deployment's api_key.
This commit is contained in:
mateo-berri 2026-10-28 03:39:58 -07:00
parent e44c8de305
commit 1009886009
4 changed files with 113 additions and 22 deletions

View file

@ -1,8 +1,10 @@
from collections.abc import Mapping
from typing import Final
import httpx
import litellm
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
@ -28,6 +30,12 @@ class VLLMError(BaseLLMException):
)
def _auth_headers(api_key: str | None) -> Mapping[str, str]:
if api_key is None:
return {}
return {"x-api-key": api_key, "Authorization": f"Bearer {api_key}"}
class VLLMModelInfo(BaseLLMModelInfo):
def validate_environment(
self,
@ -39,39 +47,51 @@ class VLLMModelInfo(BaseLLMModelInfo):
api_key: str | None = None,
api_base: str | None = None,
) -> dict:
if api_key is not None:
headers["x-api-key"] = api_key
headers["Authorization"] = f"Bearer {api_key}"
headers.update(_auth_headers(api_key))
return headers
@staticmethod
def get_api_base(api_base: str | None = None) -> str:
api_base = api_base or get_secret_str("VLLM_API_BASE")
if api_base is None:
raise ValueError("VLLM_API_BASE is not set.")
raise ValueError(
"VLLM_API_BASE is not set. Please set the environment variable, to use VLLM's pass-through - `{LITELLM_API_BASE}/vllm/{endpoint}`."
)
return api_base
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return None
return api_key
@staticmethod
def get_base_model(model: str) -> str | None:
return model
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
api_base = VLLMModelInfo.get_api_base(api_base)
endpoint: Final = "/v1/models"
url: Final = add_path_to_api_base(api_base, endpoint)
headers: Final = {"x-api-key": api_key, "Authorization": f"Bearer {api_key}"} if api_key is not None else None
response: Final = litellm.module_level_client.get(url=url, headers=headers)
response.raise_for_status()
models: Final = response.json()["data"]
return [model["id"] for model in models]
def get_models(
self, api_key: str | None = None, api_base: str | None = None
) -> list[str]: # mutable-ok: mirrors override contract
return []
def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
return VLLMError(status_code=status_code, message=error_message, headers=headers)
class HostedVLLMModelInfo(VLLMModelInfo):
@staticmethod
def get_api_base(api_base: str | None = None) -> str:
resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE")
if resolved_api_base is None:
raise ValueError("HOSTED_VLLM_API_BASE is not set.")
return resolved_api_base
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return api_key or get_secret_str("HOSTED_VLLM_API_KEY")
def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
url: Final = add_path_to_api_base(self.get_api_base(api_base), "/v1/models")
response: Final = litellm.module_level_client.get(
url=url, headers=dict(_auth_headers(self.get_api_key(api_key))), timeout=NON_LLM_CONNECTION_TIMEOUT
)
response.raise_for_status()
return [model["id"] for model in response.json()["data"]]

View file

@ -10049,12 +10049,14 @@ class ProviderConfigManager:
from litellm.llms.ollama.common_utils import OllamaModelInfo
return OllamaModelInfo()
elif LlmProviders.VLLM == provider or LlmProviders.HOSTED_VLLM == provider:
from litellm.llms.vllm.common_utils import (
VLLMModelInfo, # experimental approach, to reduce bloat on __init__.py
)
elif LlmProviders.VLLM == provider:
from litellm.llms.vllm.common_utils import VLLMModelInfo
return VLLMModelInfo()
elif LlmProviders.HOSTED_VLLM == provider:
from litellm.llms.vllm.common_utils import HostedVLLMModelInfo
return HostedVLLMModelInfo()
elif LlmProviders.LEMONADE == provider:
return litellm.LemonadeChatConfig()
elif LlmProviders.CLARIFAI == provider:

View file

@ -51,3 +51,24 @@ async def test_allm_passthrough_route_hosted_vllm_sends_normalized_model(respx_m
outbound: Final = json.loads(route.calls[0].request.content)
assert outbound["model"] == "my-deployment"
assert outbound["messages"] == [{"role": "user", "content": "Hello"}]
@pytest.mark.asyncio
async def test_allm_passthrough_route_hosted_vllm_sends_the_deployment_api_key(respx_mock: respx.MockRouter) -> None:
route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock(
return_value=httpx.Response(200, json={"ok": True})
)
response: Final = await allm_passthrough_route(
method="POST",
endpoint="v1/chat/completions",
model="hosted_vllm/my-deployment",
api_base=_API_BASE,
api_key="deployment-key",
json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]},
client=AsyncHTTPHandler(),
)
assert response.status_code == 200
sent_headers: Final = route.calls[0].request.headers
assert sent_headers["authorization"] == "Bearer deployment-key"
assert sent_headers["x-api-key"] == "deployment-key"

View file

@ -481,6 +481,54 @@ def test_hosted_vllm_wildcard_sends_the_configured_api_key_as_a_bearer_token(mon
assert models_route.call_count == 1
@respx.mock
def test_hosted_vllm_wildcard_without_api_base_lists_the_server_chat_would_call(monkeypatch: pytest.MonkeyPatch) -> None:
import litellm
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm-env.test:8000/v1")
monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-env-key")
monkeypatch.setenv("VLLM_API_BASE", "http://vllm-passthrough-env.test:8000")
passthrough_server_route: Final = respx.get("http://vllm-passthrough-env.test:8000/v1/models").mock(
return_value=httpx.Response(200, json={"data": [{"id": "model-on-the-passthrough-server"}]})
)
respx.get(
"http://hosted-vllm-env.test:8000/v1/models", headers={"Authorization": "Bearer hosted-vllm-env-key"}
).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}))
listed: Final = get_known_models_from_wildcard(
"hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", custom_llm_provider="hosted_vllm")
)
assert listed == ["hosted_vllm/qwen2.5"]
assert not passthrough_server_route.called
@respx.mock
def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_hung_server(
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm
from litellm.constants import NON_LLM_CONNECTION_TIMEOUT
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
models_route: Final = respx.get("http://vllm-timeout.test:8000/v1/models").mock(
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
)
get_known_models_from_wildcard(
"hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-timeout.test:8000/v1")
)
read_timeout: Final = models_route.calls.last.request.extensions["timeout"]["read"]
assert read_timeout is not None
assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT
def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch):
"""Regression test for #30358.