From d90200003b22358645308f9a314740cc02a497f5 Mon Sep 17 00:00:00 2001 From: jesus Date: Sun, 6 Sep 2026 18:53:32 +0000 Subject: [PATCH 1/5] fix(proxy): expand wildcard models for providers with model info but no cost map entry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/model_checks.py | 14 +++++- .../proxy/auth/test_model_checks.py | 43 ++++++++++++++++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index de2ca4762f1..55b4670d14e 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -13,7 +13,7 @@ from litellm.router import Router from litellm.router_utils.fallback_event_handlers import get_fallback_model_group from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders -from litellm.utils import get_valid_models +from litellm.utils import ProviderConfigManager, get_valid_models _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) @@ -35,6 +35,16 @@ def _check_wildcard_routing(model: str) -> bool: return False +def _provider_supports_model_discovery(provider: str) -> bool: + if provider in litellm.models_by_provider: + return True + try: + llm_provider: Final = LlmProviders(provider) + except ValueError: + return False + return ProviderConfigManager.get_provider_model_info(model=None, provider=llm_provider) is not None + + def get_provider_models(provider: str, litellm_params: LiteLLM_Params | None = None) -> list[str] | None: """ Returns the list of known models by provider @@ -42,7 +52,7 @@ def get_provider_models(provider: str, litellm_params: LiteLLM_Params | None = N if provider == "*": return get_valid_models(litellm_params=litellm_params) - if provider in litellm.models_by_provider: + if _provider_supports_model_discovery(provider): provider_models: Final = get_valid_models(custom_llm_provider=provider, litellm_params=litellm_params) return provider_models return None diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 36bfc4c5dd3..f580df2dad6 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -1,4 +1,4 @@ -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -432,6 +432,47 @@ def test_wildcard_credential_hydration_preserves_deployment_params( } +def test_get_known_models_from_wildcard_hosted_vllm_uses_provider_endpoint(): + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + response = MagicMock() + response.json.return_value = { + "data": [ + {"id": "meta-llama/Llama-3.1-8B-Instruct"}, + {"id": "qwen2.5"}, + ] + } + original_check_provider_endpoint = litellm.check_provider_endpoint + try: + litellm.check_provider_endpoint = True # test-quality-ok: required to exercise provider endpoint discovery + with ( + patch("litellm.module_level_client.get", return_value=response), # test-quality-ok: required HTTP boundary + patch( # test-quality-ok: VLLM model listing has no injectable API key seam + "litellm.llms.vllm.common_utils.VLLMModelInfo.get_api_key", + return_value="test-key", + ), + ): + result = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params(model="hosted_vllm/*", api_base="http://localhost:8000/v1"), + ) + finally: + litellm.check_provider_endpoint = original_check_provider_endpoint # test-quality-ok: restore test global + + assert result == [ + "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct", + "hosted_vllm/qwen2.5", + ] + + +def test_get_known_models_from_wildcard_unknown_provider_returns_empty(): + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + assert get_known_models_from_wildcard("not_a_real_provider/*") == [] + + def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): """Regression test for #30358. From 2ed624550d872f043ceb22ebb14797e09f4ab94f Mon Sep 17 00:00:00 2001 From: jesus Date: Sun, 6 Sep 2026 18:57:51 +0000 Subject: [PATCH 2/5] fix(vllm): stop requiring an api key to list vllm models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/vllm/common_utils.py | 9 +-------- tests/test_litellm/proxy/auth/test_model_checks.py | 4 ---- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index 2da2269e1f9..f663ec7870a 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -47,9 +47,7 @@ class VLLMModelInfo(BaseLLMModelInfo): def get_api_base(api_base: str | None = None) -> str | None: api_base = api_base or get_secret_str("VLLM_API_BASE") if api_base is None: - raise ValueError( - "VLLM_API_BASE is not set. Please set the environment variable, to use VLLM's pass-through - `{LITELLM_API_BASE}/vllm/{endpoint}`." - ) + raise ValueError("VLLM_API_BASE is not set.") return api_base @staticmethod @@ -62,12 +60,7 @@ class VLLMModelInfo(BaseLLMModelInfo): def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: api_base = VLLMModelInfo.get_api_base(api_base) - api_key = VLLMModelInfo.get_api_key(api_key) endpoint: Final = "/v1/models" - if api_base is None or api_key is None: - raise ValueError( - "VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint." - ) url: Final = _add_path_to_api_base(api_base, endpoint) response: Final = litellm.module_level_client.get( diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index f580df2dad6..451f4eac9a0 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -449,10 +449,6 @@ def test_get_known_models_from_wildcard_hosted_vllm_uses_provider_endpoint(): litellm.check_provider_endpoint = True # test-quality-ok: required to exercise provider endpoint discovery with ( patch("litellm.module_level_client.get", return_value=response), # test-quality-ok: required HTTP boundary - patch( # test-quality-ok: VLLM model listing has no injectable API key seam - "litellm.llms.vllm.common_utils.VLLMModelInfo.get_api_key", - return_value="test-key", - ), ): result = get_known_models_from_wildcard( "hosted_vllm/*", From 624f4281aac239aa45ed291ed70de24a77ad3415 Mon Sep 17 00:00:00 2001 From: jesus Date: Sun, 6 Sep 2026 19:17:12 +0000 Subject: [PATCH 3/5] fix(proxy): preserve model ids during wildcard expansion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/model_checks.py | 2 +- tests/test_litellm/proxy/auth/test_model_checks.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 55b4670d14e..fb925a3b133 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -325,7 +325,7 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_ # Only strip the leading segment when it is a known provider, so ids whose first # segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it. leading, sep, model_suffix = model.partition("/") - if sep and leading in known_providers: + if sep and leading in known_providers and (provider in litellm.models_by_provider or leading == provider): model = f"{wildcard_provider_prefix}/{model_suffix}" else: model = f"{wildcard_provider_prefix}/{model}" diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 451f4eac9a0..bea46a3fd23 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -442,6 +442,7 @@ def test_get_known_models_from_wildcard_hosted_vllm_uses_provider_endpoint(): "data": [ {"id": "meta-llama/Llama-3.1-8B-Instruct"}, {"id": "qwen2.5"}, + {"id": "openai/foo"}, ] } original_check_provider_endpoint = litellm.check_provider_endpoint @@ -460,6 +461,7 @@ def test_get_known_models_from_wildcard_hosted_vllm_uses_provider_endpoint(): assert result == [ "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct", "hosted_vllm/qwen2.5", + "hosted_vllm/openai/foo", ] From 9edc1018fbe830d0e14e74e07b9aa0d27101aedc Mon Sep 17 00:00:00 2001 From: jesus Date: Sun, 6 Sep 2026 19:47:34 +0000 Subject: [PATCH 4/5] fix(vllm): forward api key for model discovery Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/vllm/common_utils.py | 9 ++++-- .../proxy/auth/test_model_checks.py | 28 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index f663ec7870a..106aae29aab 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -63,8 +63,13 @@ class VLLMModelInfo(BaseLLMModelInfo): endpoint: Final = "/v1/models" url: Final = _add_path_to_api_base(api_base, endpoint) - response: Final = litellm.module_level_client.get( - url=url, + response: Final = ( + litellm.module_level_client.get( + url=url, + headers={"x-api-key": api_key}, # mutable-ok: optional authentication header + ) + if api_key is not None + else litellm.module_level_client.get(url=url) ) response.raise_for_status() diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index bea46a3fd23..021d6a7f056 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -465,6 +465,34 @@ def test_get_known_models_from_wildcard_hosted_vllm_uses_provider_endpoint(): ] +def test_get_known_models_from_wildcard_hosted_vllm_forwards_api_key(): + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + response = MagicMock() + response.json.return_value = {"data": [{"id": "qwen2.5"}]} + original_check_provider_endpoint = litellm.check_provider_endpoint + try: + litellm.check_provider_endpoint = True # test-quality-ok: required to exercise provider endpoint discovery + with patch( # test-quality-ok: required HTTP boundary + "litellm.module_level_client.get", return_value=response + ) as mock_get: + result = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params( + model="hosted_vllm/*", + api_base="http://localhost:8000/v1", + api_key="test-key", + ), + ) + finally: + litellm.check_provider_endpoint = original_check_provider_endpoint # test-quality-ok: restore test global + + assert result == ["hosted_vllm/qwen2.5"] + assert mock_get.call_args.kwargs["headers"] == {"x-api-key": "test-key"} + + def test_get_known_models_from_wildcard_unknown_provider_returns_empty(): from litellm.proxy.auth.model_checks import get_known_models_from_wildcard From 519b979b23b23f428950ed98a0c97a97ee5d3ba9 Mon Sep 17 00:00:00 2001 From: jesus Date: Sun, 6 Sep 2026 20:15:07 +0000 Subject: [PATCH 5/5] fix(vllm): support bearer auth for model discovery Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/vllm/common_utils.py | 6 +++++- tests/test_litellm/proxy/auth/test_model_checks.py | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index 106aae29aab..f1ad1b0ce11 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -41,6 +41,7 @@ class VLLMModelInfo(BaseLLMModelInfo): ) -> dict: if api_key is not None: headers["x-api-key"] = api_key + headers["Authorization"] = f"Bearer {api_key}" return headers @staticmethod @@ -66,7 +67,10 @@ class VLLMModelInfo(BaseLLMModelInfo): response: Final = ( litellm.module_level_client.get( url=url, - headers={"x-api-key": api_key}, # mutable-ok: optional authentication header + headers={ # mutable-ok: optional authentication headers + "x-api-key": api_key, + "Authorization": f"Bearer {api_key}", + }, ) if api_key is not None else litellm.module_level_client.get(url=url) diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 021d6a7f056..83e6e46ea1a 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -490,7 +490,10 @@ def test_get_known_models_from_wildcard_hosted_vllm_forwards_api_key(): litellm.check_provider_endpoint = original_check_provider_endpoint # test-quality-ok: restore test global assert result == ["hosted_vllm/qwen2.5"] - assert mock_get.call_args.kwargs["headers"] == {"x-api-key": "test-key"} + assert mock_get.call_args.kwargs["headers"] == { + "x-api-key": "test-key", + "Authorization": "Bearer test-key", + } def test_get_known_models_from_wildcard_unknown_provider_returns_empty():