From 2c1e7baa68c516ee8ce65e08ae917203872cf29b Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Sat, 24 Oct 2026 14:32:51 -0700 Subject: [PATCH 1/2] fix(proxy): list the models behind custom_openai/* and hosted_vllm/* wildcards on /v1/models custom_openai had no model lister, so check_provider_endpoint never asked the server what it serves. The vLLM lister demanded a key and then always threw it away, so it raised on every call. Both now ask the server chat calls go to, with the key chat calls send. --- litellm/llms/openai_like/model_info.py | 64 ++++++++++++++++++ litellm/llms/vllm/common_utils.py | 27 ++++---- litellm/responses/bridge_check.py | 14 ++-- litellm/utils.py | 4 ++ tests/unit/proxy/auth/test_model_checks.py | 77 ++++++++++++++++++++++ 5 files changed, 165 insertions(+), 21 deletions(-) diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py index e3b2dc427c4..4e1cc87c1d5 100644 --- a/litellm/llms/openai_like/model_info.py +++ b/litellm/llms/openai_like/model_info.py @@ -7,10 +7,15 @@ from typing import Annotated, Final, TypeAlias import httpx from pydantic import BeforeValidator, ConfigDict +import litellm from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.secret_managers.main import get_secret_str from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.llms.openai import AllMessageValues from litellm.utils import add_path_to_api_base MODEL_INFO_REFRESH_SECONDS: Final = 300 @@ -91,3 +96,62 @@ async def get_openai_compatible_model_info( cache.set_cache(cache_key, models, ttl=MODEL_INFO_REFRESH_SECONDS) return next((card.token_limits() for card in models.data if card.id == model), _EMPTY_LIMITS) + + +def list_openai_compatible_model_ids( + *, models_url: str, api_key: str | None +) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override + """ + The model ids an OpenAI-compatible server lists at `models_url`. + Raises on a non-2xx answer so wildcard discovery logs the failure instead of showing an empty list as success. + """ + response: Final = litellm.module_level_client.get( + url=models_url, + headers={"Authorization": f"Bearer {api_key}"} if api_key else None, + follow_redirects=False, + ) + response.raise_for_status() + return [card.id for card in _ModelList.model_validate_json(response.content).data] + + +class CustomOpenAIModelInfo(BaseLLMModelInfo): + """ + Lists the models behind a `custom_openai/*` wildcard by asking the server chat calls go to, with the key chat + calls send, so `/v1/models` and inference can't disagree about where that server is or how to reach it. + """ + + @staticmethod + def get_api_base(api_base: str | None = None) -> str | None: + from litellm.responses.bridge_check import resolve_openai_api_base + + return resolve_openai_api_base(api_base) + + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: mirrors override contract + model: str, + messages: list[AllMessageValues], + optional_params: dict[str, object], # mutable-ok: mirrors override contract + litellm_params: dict[str, object], # mutable-ok: mirrors override contract + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: mirrors override contract + return headers + + @staticmethod + def get_base_model(model: str) -> str | None: + return model + + def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + resolved_api_base: Final = self.get_api_base(api_base) + if resolved_api_base is None: + raise ValueError("custom_openai needs an api_base to list models from") + # Chat calls go to `{api_base}/chat/completions` (the OpenAI client's base_url contract), so the list lives at + # `{api_base}/models`; adding `/v1` here would list models from a path chat never calls. + return list_openai_compatible_model_ids( + models_url=join_api_base(resolved_api_base, "models"), api_key=self.get_api_key(api_key) + ) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index 6484cdc88e3..e2b807e7250 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -2,9 +2,9 @@ from typing import Final import httpx -import litellm from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.openai_like.model_info import list_openai_compatible_model_ids from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.utils import add_path_to_api_base @@ -61,24 +61,19 @@ class VLLMModelInfo(BaseLLMModelInfo): return model def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: - api_base = VLLMModelInfo.get_api_base(api_base) - api_key = VLLMModelInfo.get_api_key(api_key) - endpoint: Final = "/v1/models" - if api_base is None or api_key is None: + """ + Reads the server and key the same way `hosted_vllm` chat calls do. `get_api_key` above always answers None + (the passthrough route sends no key), so model listing must not lean on it or every listing fails. + """ + resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE") or get_secret_str("VLLM_API_BASE") + if resolved_api_base is None: raise ValueError( - "VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint." + "Set api_base (or HOSTED_VLLM_API_BASE / VLLM_API_BASE) to list the models a vLLM server serves." ) - - url: Final = add_path_to_api_base(api_base, endpoint) - response: Final = litellm.module_level_client.get( - url=url, + return list_openai_compatible_model_ids( + models_url=add_path_to_api_base(resolved_api_base, "/v1/models"), + api_key=api_key or get_secret_str("HOSTED_VLLM_API_KEY"), ) - response.raise_for_status() - - models: Final = response.json()["data"] - - return [model["id"] for model in models] - def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return VLLMError(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 60318d6b184..cf6ece1e7ea 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -68,11 +68,12 @@ def _is_openai_backed_api_base(api_base: str) -> bool: return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}")) -def _resolve_openai_api_base(api_base: str | None) -> str: - """Effective OpenAI base a chat request will hit: arg > global > env > default. The bridge gate - and the ``_complete_custom_openai`` chat handler MUST resolve this identically, or a custom base - set via ``litellm.api_base`` or ``OPENAI_BASE_URL``/``OPENAI_API_BASE`` is invisible to the gate, - which then misreads it as the default OpenAI endpoint and bridges a request the backend can't serve.""" +def resolve_openai_api_base(api_base: str | None) -> str: + """Effective OpenAI base a chat request will hit: arg > global > env > default. The bridge gate, + the ``_complete_custom_openai`` chat handler and ``custom_openai/*`` model listing MUST resolve this + identically, or a custom base set via ``litellm.api_base`` or ``OPENAI_BASE_URL``/``OPENAI_API_BASE`` + is invisible to the gate, which then misreads it as the default OpenAI endpoint and bridges a request + the backend can't serve.""" return ( api_base or litellm.api_base @@ -82,6 +83,9 @@ def _resolve_openai_api_base(api_base: str | None) -> str: ) +_resolve_openai_api_base = resolve_openai_api_base + + def _mantle_responses_series_name(model: str) -> str | None: """The bare OpenAI series name of a Bedrock Mantle model that also serves the Responses API, or ``None``.""" from litellm.llms.bedrock_mantle.common_utils import mantle_supports_responses, split_mantle_region_prefix diff --git a/litellm/utils.py b/litellm/utils.py index 1db5f7129df..99fb68c938f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10016,6 +10016,10 @@ class ProviderConfigManager: ) return VLLMModelInfo() + elif LlmProviders.CUSTOM_OPENAI == provider: + from litellm.llms.openai_like.model_info import CustomOpenAIModelInfo + + return CustomOpenAIModelInfo() elif LlmProviders.LEMONADE == provider: return litellm.LemonadeChatConfig() elif LlmProviders.CLARIFAI == provider: diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 83761620cc9..546af613ad0 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1521,3 +1521,80 @@ def test_prefixed_alias_over_a_bare_catch_all_lists_ids_that_route_through_it(mo ] assert f"team-partial-route/{_ROUTE_OPENAI_ID}" in v1_models assert unrouted_ids == [] + + +_SERVER_MODEL_IDS: Final = ("Qwen/Qwen3-8B", "meta-llama/Llama-3.1-8B-Instruct") + + +@pytest.mark.parametrize( + ("provider", "api_key", "expected_authorization"), + ( + ("custom_openai", "sk-custom", "Bearer sk-custom"), + ("hosted_vllm", "sk-vllm", "Bearer sk-vllm"), + ("hosted_vllm", None, None), + ), +) +def test_openai_compatible_server_wildcard_lists_the_models_that_server_serves( + monkeypatch: pytest.MonkeyPatch, provider: str, api_key: str | None, expected_authorization: str | None +) -> None: + """Regression test for #20064: `/v1/models` asks the same server, with the same key, that chat calls use.""" + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + api_base: Final = f"https://{provider}-{api_key}.test/tenant/v1" + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.get(f"{api_base}/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": model_id} for model_id in _SERVER_MODEL_IDS]}) + ) + result: Final = get_known_models_from_wildcard( + wildcard_model="my-server/*", + litellm_params=LiteLLM_Params(model=f"{provider}/*", api_base=api_base, api_key=api_key), + ) + + assert route.calls.last.request.headers.get("authorization") == expected_authorization + assert result == [f"my-server/{model_id}" for model_id in _SERVER_MODEL_IDS] + + +@respx.mock(assert_all_called=True) +def test_custom_openai_wildcard_lists_models_beside_the_chat_route_when_the_base_has_no_v1( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + """Chat calls go to `{api_base}/chat/completions`, so the model list must come from `{api_base}/models`.""" + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + api_base: Final = "https://custom-openai-no-v1.test/serving" + respx_mock.get(f"{api_base}/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": model_id} for model_id in _SERVER_MODEL_IDS]}) + ) + + result: Final = get_known_models_from_wildcard( + wildcard_model="my-server/*", + litellm_params=LiteLLM_Params(model="custom_openai/*", api_base=api_base, api_key="sk-custom"), + ) + + assert result == [f"my-server/{model_id}" for model_id in _SERVER_MODEL_IDS] + + +def test_openai_compatible_server_wildcard_lists_nothing_when_the_server_rejects_the_listing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + api_base: Final = "https://custom-openai-rejects.test/v1" + with respx.mock(assert_all_called=True) as respx_mock: + respx_mock.get(f"{api_base}/models").mock(return_value=httpx.Response(401, json={"error": "bad key"})) + result: Final = get_known_models_from_wildcard( + wildcard_model="my-server/*", + litellm_params=LiteLLM_Params(model="custom_openai/*", api_base=api_base, api_key="sk-wrong"), + ) + + assert result == [] From d9f25782c65862f6270e4748913f99c23e46af01 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Mon, 26 Oct 2026 17:43:52 -0700 Subject: [PATCH 2/2] chore(lint): keep the type-discipline counts flat for the model lister --- litellm/llms/openai_like/model_info.py | 6 ++++-- litellm/responses/bridge_check.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/llms/openai_like/model_info.py b/litellm/llms/openai_like/model_info.py index 4e1cc87c1d5..e87074bfec6 100644 --- a/litellm/llms/openai_like/model_info.py +++ b/litellm/llms/openai_like/model_info.py @@ -134,7 +134,7 @@ class CustomOpenAIModelInfo(BaseLLMModelInfo): self, headers: dict[str, str], # mutable-ok: mirrors override contract model: str, - messages: list[AllMessageValues], + messages: list[AllMessageValues], # mutable-ok: mirrors override contract optional_params: dict[str, object], # mutable-ok: mirrors override contract litellm_params: dict[str, object], # mutable-ok: mirrors override contract api_key: str | None = None, @@ -146,7 +146,9 @@ class CustomOpenAIModelInfo(BaseLLMModelInfo): def get_base_model(model: str) -> str | None: return model - def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + def get_models( + self, api_key: str | None = None, api_base: str | None = None + ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override resolved_api_base: Final = self.get_api_base(api_base) if resolved_api_base is None: raise ValueError("custom_openai needs an api_base to list models from") diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index cf6ece1e7ea..147d0ab098f 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -83,7 +83,7 @@ def resolve_openai_api_base(api_base: str | None) -> str: ) -_resolve_openai_api_base = resolve_openai_api_base +_resolve_openai_api_base: Final = resolve_openai_api_base def _mantle_responses_series_name(model: str) -> str | None: