From 1009886009b3944c37329fac210dfd815c611f62 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:39:58 -0700 Subject: [PATCH] fix(vllm): list hosted_vllm models from the server chat uses, with a bounded wait The hosted_vllm listing now resolves HOSTED_VLLM_API_BASE and HOSTED_VLLM_API_KEY like hosted_vllm chat, instead of the /vllm pass-through's VLLM_API_BASE, and waits at most NON_LLM_CONNECTION_TIMEOUT rather than the completion timeout. vllm/* keeps listing nothing, since its chat runs in process. The router pass-through now forwards the deployment's api_key. --- litellm/llms/vllm/common_utils.py | 56 +++++++++++++++------- litellm/utils.py | 10 ++-- tests/unit/passthrough/test_main.py | 21 ++++++++ tests/unit/proxy/auth/test_model_checks.py | 48 +++++++++++++++++++ 4 files changed, 113 insertions(+), 22 deletions(-) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index 3bd83387db2..edb3779a159 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -1,8 +1,10 @@ +from collections.abc import Mapping from typing import Final import httpx import litellm +from litellm.constants import NON_LLM_CONNECTION_TIMEOUT from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str @@ -28,6 +30,12 @@ class VLLMError(BaseLLMException): ) +def _auth_headers(api_key: str | None) -> Mapping[str, str]: + if api_key is None: + return {} + return {"x-api-key": api_key, "Authorization": f"Bearer {api_key}"} + + class VLLMModelInfo(BaseLLMModelInfo): def validate_environment( self, @@ -39,39 +47,51 @@ class VLLMModelInfo(BaseLLMModelInfo): api_key: str | None = None, api_base: str | None = None, ) -> dict: - if api_key is not None: - headers["x-api-key"] = api_key - headers["Authorization"] = f"Bearer {api_key}" + headers.update(_auth_headers(api_key)) return headers @staticmethod def get_api_base(api_base: str | None = None) -> str: api_base = api_base or get_secret_str("VLLM_API_BASE") if api_base is None: - raise ValueError("VLLM_API_BASE is not set.") + raise ValueError( + "VLLM_API_BASE is not set. Please set the environment variable, to use VLLM's pass-through - `{LITELLM_API_BASE}/vllm/{endpoint}`." + ) return api_base @staticmethod def get_api_key(api_key: str | None = None) -> str | None: - return None + return api_key @staticmethod def get_base_model(model: str) -> str | None: return model - def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: - api_base = VLLMModelInfo.get_api_base(api_base) - endpoint: Final = "/v1/models" - - url: Final = add_path_to_api_base(api_base, endpoint) - headers: Final = {"x-api-key": api_key, "Authorization": f"Bearer {api_key}"} if api_key is not None else None - response: Final = litellm.module_level_client.get(url=url, headers=headers) - - response.raise_for_status() - - models: Final = response.json()["data"] - - return [model["id"] for model in models] + def get_models( + self, api_key: str | None = None, api_base: str | None = None + ) -> list[str]: # mutable-ok: mirrors override contract + return [] def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return VLLMError(status_code=status_code, message=error_message, headers=headers) + + +class HostedVLLMModelInfo(VLLMModelInfo): + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE") + if resolved_api_base is None: + raise ValueError("HOSTED_VLLM_API_BASE is not set.") + return resolved_api_base + + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return api_key or get_secret_str("HOSTED_VLLM_API_KEY") + + def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + url: Final = add_path_to_api_base(self.get_api_base(api_base), "/v1/models") + response: Final = litellm.module_level_client.get( + url=url, headers=dict(_auth_headers(self.get_api_key(api_key))), timeout=NON_LLM_CONNECTION_TIMEOUT + ) + response.raise_for_status() + return [model["id"] for model in response.json()["data"]] diff --git a/litellm/utils.py b/litellm/utils.py index 1e095966a43..48c50969058 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10049,12 +10049,14 @@ class ProviderConfigManager: from litellm.llms.ollama.common_utils import OllamaModelInfo return OllamaModelInfo() - elif LlmProviders.VLLM == provider or LlmProviders.HOSTED_VLLM == provider: - from litellm.llms.vllm.common_utils import ( - VLLMModelInfo, # experimental approach, to reduce bloat on __init__.py - ) + elif LlmProviders.VLLM == provider: + from litellm.llms.vllm.common_utils import VLLMModelInfo return VLLMModelInfo() + elif LlmProviders.HOSTED_VLLM == provider: + from litellm.llms.vllm.common_utils import HostedVLLMModelInfo + + return HostedVLLMModelInfo() elif LlmProviders.LEMONADE == provider: return litellm.LemonadeChatConfig() elif LlmProviders.CLARIFAI == provider: diff --git a/tests/unit/passthrough/test_main.py b/tests/unit/passthrough/test_main.py index 78fa83d126b..9c07729da86 100644 --- a/tests/unit/passthrough/test_main.py +++ b/tests/unit/passthrough/test_main.py @@ -51,3 +51,24 @@ async def test_allm_passthrough_route_hosted_vllm_sends_normalized_model(respx_m outbound: Final = json.loads(route.calls[0].request.content) assert outbound["model"] == "my-deployment" assert outbound["messages"] == [{"role": "user", "content": "Hello"}] + + +@pytest.mark.asyncio +async def test_allm_passthrough_route_hosted_vllm_sends_the_deployment_api_key(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + response: Final = await allm_passthrough_route( + method="POST", + endpoint="v1/chat/completions", + model="hosted_vllm/my-deployment", + api_base=_API_BASE, + api_key="deployment-key", + json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, + client=AsyncHTTPHandler(), + ) + assert response.status_code == 200 + sent_headers: Final = route.calls[0].request.headers + assert sent_headers["authorization"] == "Bearer deployment-key" + assert sent_headers["x-api-key"] == "deployment-key" + diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 490222361d2..623eed1ba7b 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -481,6 +481,54 @@ def test_hosted_vllm_wildcard_sends_the_configured_api_key_as_a_bearer_token(mon assert models_route.call_count == 1 +@respx.mock +def test_hosted_vllm_wildcard_without_api_base_lists_the_server_chat_would_call(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm-env.test:8000/v1") + monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-env-key") + monkeypatch.setenv("VLLM_API_BASE", "http://vllm-passthrough-env.test:8000") + passthrough_server_route: Final = respx.get("http://vllm-passthrough-env.test:8000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "model-on-the-passthrough-server"}]}) + ) + respx.get( + "http://hosted-vllm-env.test:8000/v1/models", headers={"Authorization": "Bearer hosted-vllm-env-key"} + ).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + + listed: Final = get_known_models_from_wildcard( + "hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", custom_llm_provider="hosted_vllm") + ) + + assert listed == ["hosted_vllm/qwen2.5"] + assert not passthrough_server_route.called + + +@respx.mock +def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_hung_server( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.constants import NON_LLM_CONNECTION_TIMEOUT + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + models_route: Final = respx.get("http://vllm-timeout.test:8000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + + get_known_models_from_wildcard( + "hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-timeout.test:8000/v1") + ) + + read_timeout: Final = models_route.calls.last.request.extensions["timeout"]["read"] + assert read_timeout is not None + assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT + + def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): """Regression test for #30358.