Merge 519b979b23 into eddfb5fb20

litellm/llms/vllm/common_utils.py

@@ -41,15 +41,14 @@ class VLLMModelInfo(BaseLLMModelInfo):
     ) -> dict:
         if api_key is not None:
             headers["x-api-key"] = api_key
+            headers["Authorization"] = f"Bearer {api_key}"
         return headers
 
     @staticmethod
     def get_api_base(api_base: str | None = None) -> str | None:
         api_base = api_base or get_secret_str("VLLM_API_BASE")
         if api_base is None:
-            raise ValueError(
-                "VLLM_API_BASE is not set. Please set the environment variable, to use VLLM's pass-through - `{LITELLM_API_BASE}/vllm/{endpoint}`."
-            )
+            raise ValueError("VLLM_API_BASE is not set.")
         return api_base
 
     @staticmethod
@@ -62,16 +61,19 @@ class VLLMModelInfo(BaseLLMModelInfo):
 
     def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
         api_base = VLLMModelInfo.get_api_base(api_base)
-        api_key = VLLMModelInfo.get_api_key(api_key)
         endpoint: Final = "/v1/models"
-        if api_base is None or api_key is None:
-            raise ValueError(
-                "VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint."
-            )
 
         url: Final = _add_path_to_api_base(api_base, endpoint)
-        response: Final = litellm.module_level_client.get(
-            url=url,
+        response: Final = (
+            litellm.module_level_client.get(
+                url=url,
+                headers={  # mutable-ok: optional authentication headers
+                    "x-api-key": api_key,
+                    "Authorization": f"Bearer {api_key}",
+                },
+            )
+            if api_key is not None
+            else litellm.module_level_client.get(url=url)
         )
 
         response.raise_for_status()

litellm/proxy/auth/model_checks.py

@@ -13,7 +13,7 @@ from litellm.router import Router
 from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
 from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
 from litellm.types.utils import LlmProviders
-from litellm.utils import get_valid_models
+from litellm.utils import ProviderConfigManager, get_valid_models
 
 _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
 
@@ -35,6 +35,16 @@ def _check_wildcard_routing(model: str) -> bool:
     return False
 
 
+def _provider_supports_model_discovery(provider: str) -> bool:
+    if provider in litellm.models_by_provider:
+        return True
+    try:
+        llm_provider: Final = LlmProviders(provider)
+    except ValueError:
+        return False
+    return ProviderConfigManager.get_provider_model_info(model=None, provider=llm_provider) is not None
+
+
 def get_provider_models(provider: str, litellm_params: LiteLLM_Params | None = None) -> list[str] | None:
     """
     Returns the list of known models by provider
@@ -42,7 +52,7 @@ def get_provider_models(provider: str, litellm_params: LiteLLM_Params | None = N
     if provider == "*":
         return get_valid_models(litellm_params=litellm_params)
 
-    if provider in litellm.models_by_provider:
+    if _provider_supports_model_discovery(provider):
         provider_models: Final = get_valid_models(custom_llm_provider=provider, litellm_params=litellm_params)
         return provider_models
     return None
@@ -315,7 +325,7 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_
             # Only strip the leading segment when it is a known provider, so ids whose first
             # segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it.
             leading, sep, model_suffix = model.partition("/")
-            if sep and leading in known_providers:
+            if sep and leading in known_providers and (provider in litellm.models_by_provider or leading == provider):
                 model = f"{wildcard_provider_prefix}/{model_suffix}"
             else:
                 model = f"{wildcard_provider_prefix}/{model}"

tests/test_litellm/proxy/auth/test_model_checks.py

@@ -1,4 +1,4 @@
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
@@ -432,6 +432,76 @@ def test_wildcard_credential_hydration_preserves_deployment_params(
     }
 
 
+def test_get_known_models_from_wildcard_hosted_vllm_uses_provider_endpoint():
+    import litellm
+    from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
+    from litellm.types.router import LiteLLM_Params
+
+    response = MagicMock()
+    response.json.return_value = {
+        "data": [
+            {"id": "meta-llama/Llama-3.1-8B-Instruct"},
+            {"id": "qwen2.5"},
+            {"id": "openai/foo"},
+        ]
+    }
+    original_check_provider_endpoint = litellm.check_provider_endpoint
+    try:
+        litellm.check_provider_endpoint = True  # test-quality-ok: required to exercise provider endpoint discovery
+        with (
+            patch("litellm.module_level_client.get", return_value=response),  # test-quality-ok: required HTTP boundary
+        ):
+            result = get_known_models_from_wildcard(
+                "hosted_vllm/*",
+                LiteLLM_Params(model="hosted_vllm/*", api_base="http://localhost:8000/v1"),
+            )
+    finally:
+        litellm.check_provider_endpoint = original_check_provider_endpoint  # test-quality-ok: restore test global
+
+    assert result == [
+        "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct",
+        "hosted_vllm/qwen2.5",
+        "hosted_vllm/openai/foo",
+    ]
+
+
+def test_get_known_models_from_wildcard_hosted_vllm_forwards_api_key():
+    import litellm
+    from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
+    from litellm.types.router import LiteLLM_Params
+
+    response = MagicMock()
+    response.json.return_value = {"data": [{"id": "qwen2.5"}]}
+    original_check_provider_endpoint = litellm.check_provider_endpoint
+    try:
+        litellm.check_provider_endpoint = True  # test-quality-ok: required to exercise provider endpoint discovery
+        with patch(  # test-quality-ok: required HTTP boundary
+            "litellm.module_level_client.get", return_value=response
+        ) as mock_get:
+            result = get_known_models_from_wildcard(
+                "hosted_vllm/*",
+                LiteLLM_Params(
+                    model="hosted_vllm/*",
+                    api_base="http://localhost:8000/v1",
+                    api_key="test-key",
+                ),
+            )
+    finally:
+        litellm.check_provider_endpoint = original_check_provider_endpoint  # test-quality-ok: restore test global
+
+    assert result == ["hosted_vllm/qwen2.5"]
+    assert mock_get.call_args.kwargs["headers"] == {
+        "x-api-key": "test-key",
+        "Authorization": "Bearer test-key",
+    }
+
+
+def test_get_known_models_from_wildcard_unknown_provider_returns_empty():
+    from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
+
+    assert get_known_models_from_wildcard("not_a_real_provider/*") == []
+
+
 def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch):
     """Regression test for #30358.