Merge 15d78099f0 into cfe0100e1f

litellm/llms/openai_like/model_info.py

@@ -7,10 +7,15 @@ from typing import Annotated, Final, TypeAlias
 import httpx
 from pydantic import BeforeValidator, ConfigDict
 
+import litellm
 from litellm._logging import verbose_logger
 from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.litellm_core_utils.url_utils import join_api_base
+from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.base import LiteLLMBaseModel
+from litellm.types.llms.openai import AllMessageValues
 from litellm.utils import add_path_to_api_base
 
 MODEL_INFO_REFRESH_SECONDS: Final = 300
@@ -91,3 +96,64 @@ async def get_openai_compatible_model_info(
 
     cache.set_cache(cache_key, models, ttl=MODEL_INFO_REFRESH_SECONDS)
     return next((card.token_limits() for card in models.data if card.id == model), _EMPTY_LIMITS)
+
+
+def list_openai_compatible_model_ids(
+    *, models_url: str, api_key: str | None
+) -> list[str]:  # mutable-ok: matches get_models' list[str] contract shared by every provider override
+    """
+    The model ids an OpenAI-compatible server lists at `models_url`.
+    Raises on a non-2xx answer so wildcard discovery logs the failure instead of showing an empty list as success.
+    """
+    response: Final = litellm.module_level_client.get(
+        url=models_url,
+        headers={"Authorization": f"Bearer {api_key}"} if api_key else None,
+        follow_redirects=False,
+    )
+    response.raise_for_status()
+    return [card.id for card in _ModelList.model_validate_json(response.content).data]
+
+
+class CustomOpenAIModelInfo(BaseLLMModelInfo):
+    """
+    Lists the models behind a `custom_openai/*` wildcard by asking the server chat calls go to, with the key chat
+    calls send, so `/v1/models` and inference can't disagree about where that server is or how to reach it.
+    """
+
+    @staticmethod
+    def get_api_base(api_base: str | None = None) -> str | None:
+        from litellm.responses.bridge_check import resolve_openai_api_base
+
+        return resolve_openai_api_base(api_base)
+
+    @staticmethod
+    def get_api_key(api_key: str | None = None) -> str | None:
+        return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
+
+    def validate_environment(
+        self,
+        headers: dict[str, str],  # mutable-ok: mirrors override contract
+        model: str,
+        messages: list[AllMessageValues],  # mutable-ok: mirrors override contract
+        optional_params: dict[str, object],  # mutable-ok: mirrors override contract
+        litellm_params: dict[str, object],  # mutable-ok: mirrors override contract
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: mirrors override contract
+        return headers
+
+    @staticmethod
+    def get_base_model(model: str) -> str | None:
+        return model
+
+    def get_models(
+        self, api_key: str | None = None, api_base: str | None = None
+    ) -> list[str]:  # mutable-ok: matches get_models' list[str] contract shared by every provider override
+        resolved_api_base: Final = self.get_api_base(api_base)
+        if resolved_api_base is None:
+            raise ValueError("custom_openai needs an api_base to list models from")
+        # Chat calls go to `{api_base}/chat/completions` (the OpenAI client's base_url contract), so the list lives at
+        # `{api_base}/models`; adding `/v1` here would list models from a path chat never calls.
+        return list_openai_compatible_model_ids(
+            models_url=join_api_base(resolved_api_base, "models"), api_key=self.get_api_key(api_key)
+        )

litellm/llms/vllm/common_utils.py

@@ -2,9 +2,9 @@ from typing import Final
 
 import httpx
 
-import litellm
 from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.openai_like.model_info import list_openai_compatible_model_ids
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
 from litellm.utils import add_path_to_api_base
@@ -61,24 +61,19 @@ class VLLMModelInfo(BaseLLMModelInfo):
         return model
 
     def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
-        api_base = VLLMModelInfo.get_api_base(api_base)
-        api_key = VLLMModelInfo.get_api_key(api_key)
-        endpoint: Final = "/v1/models"
-        if api_base is None or api_key is None:
+        """
+        Reads the server and key the same way `hosted_vllm` chat calls do. `get_api_key` above always answers None
+        (the passthrough route sends no key), so model listing must not lean on it or every listing fails.
+        """
+        resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE") or get_secret_str("VLLM_API_BASE")
+        if resolved_api_base is None:
             raise ValueError(
-                "VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint."
+                "Set api_base (or HOSTED_VLLM_API_BASE / VLLM_API_BASE) to list the models a vLLM server serves."
             )
-
-        url: Final = add_path_to_api_base(api_base, endpoint)
-        response: Final = litellm.module_level_client.get(
-            url=url,
+        return list_openai_compatible_model_ids(
+            models_url=add_path_to_api_base(resolved_api_base, "/v1/models"),
+            api_key=api_key or get_secret_str("HOSTED_VLLM_API_KEY"),
         )
 
-        response.raise_for_status()
-
-        models: Final = response.json()["data"]
-
-        return [model["id"] for model in models]
-
     def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
         return VLLMError(status_code=status_code, message=error_message, headers=headers)

litellm/responses/bridge_check.py

@@ -68,11 +68,12 @@ def _is_openai_backed_api_base(api_base: str) -> bool:
     return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}"))
 
 
-def _resolve_openai_api_base(api_base: str | None) -> str:
-    """Effective OpenAI base a chat request will hit: arg > global > env > default. The bridge gate
-    and the ``_complete_custom_openai`` chat handler MUST resolve this identically, or a custom base
-    set via ``litellm.api_base`` or ``OPENAI_BASE_URL``/``OPENAI_API_BASE`` is invisible to the gate,
-    which then misreads it as the default OpenAI endpoint and bridges a request the backend can't serve."""
+def resolve_openai_api_base(api_base: str | None) -> str:
+    """Effective OpenAI base a chat request will hit: arg > global > env > default. The bridge gate,
+    the ``_complete_custom_openai`` chat handler and ``custom_openai/*`` model listing MUST resolve this
+    identically, or a custom base set via ``litellm.api_base`` or ``OPENAI_BASE_URL``/``OPENAI_API_BASE``
+    is invisible to the gate, which then misreads it as the default OpenAI endpoint and bridges a request
+    the backend can't serve."""
     return (
         api_base
         or litellm.api_base
@@ -82,6 +83,9 @@ def _resolve_openai_api_base(api_base: str | None) -> str:
     )
 
 
+_resolve_openai_api_base: Final = resolve_openai_api_base
+
+
 def _mantle_responses_series_name(model: str) -> str | None:
     """The bare OpenAI series name of a Bedrock Mantle model that also serves the Responses API, or ``None``."""
     from litellm.llms.bedrock_mantle.common_utils import mantle_supports_responses, split_mantle_region_prefix

litellm/utils.py

@@ -10055,6 +10055,10 @@ class ProviderConfigManager:
             )
 
             return VLLMModelInfo()
+        elif LlmProviders.CUSTOM_OPENAI == provider:
+            from litellm.llms.openai_like.model_info import CustomOpenAIModelInfo
+
+            return CustomOpenAIModelInfo()
         elif LlmProviders.LEMONADE == provider:
             return litellm.LemonadeChatConfig()
         elif LlmProviders.CLARIFAI == provider:

tests/unit/proxy/auth/test_model_checks.py

@@ -1521,3 +1521,80 @@ def test_prefixed_alias_over_a_bare_catch_all_lists_ids_that_route_through_it(mo
     ]
     assert f"team-partial-route/{_ROUTE_OPENAI_ID}" in v1_models
     assert unrouted_ids == []
+
+
+_SERVER_MODEL_IDS: Final = ("Qwen/Qwen3-8B", "meta-llama/Llama-3.1-8B-Instruct")
+
+
+@pytest.mark.parametrize(
+    ("provider", "api_key", "expected_authorization"),
+    (
+        ("custom_openai", "sk-custom", "Bearer sk-custom"),
+        ("hosted_vllm", "sk-vllm", "Bearer sk-vllm"),
+        ("hosted_vllm", None, None),
+    ),
+)
+def test_openai_compatible_server_wildcard_lists_the_models_that_server_serves(
+    monkeypatch: pytest.MonkeyPatch, provider: str, api_key: str | None, expected_authorization: str | None
+) -> None:
+    """Regression test for #20064: `/v1/models` asks the same server, with the same key, that chat calls use."""
+    import litellm
+    from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
+    from litellm.types.router import LiteLLM_Params
+
+    monkeypatch.setattr(litellm, "check_provider_endpoint", True)
+    monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False)
+    api_base: Final = f"https://{provider}-{api_key}.test/tenant/v1"
+    with respx.mock(assert_all_called=True) as respx_mock:
+        route: Final = respx_mock.get(f"{api_base}/models").mock(
+            return_value=httpx.Response(200, json={"data": [{"id": model_id} for model_id in _SERVER_MODEL_IDS]})
+        )
+        result: Final = get_known_models_from_wildcard(
+            wildcard_model="my-server/*",
+            litellm_params=LiteLLM_Params(model=f"{provider}/*", api_base=api_base, api_key=api_key),
+        )
+
+    assert route.calls.last.request.headers.get("authorization") == expected_authorization
+    assert result == [f"my-server/{model_id}" for model_id in _SERVER_MODEL_IDS]
+
+
+@respx.mock(assert_all_called=True)
+def test_custom_openai_wildcard_lists_models_beside_the_chat_route_when_the_base_has_no_v1(
+    monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter
+) -> None:
+    """Chat calls go to `{api_base}/chat/completions`, so the model list must come from `{api_base}/models`."""
+    import litellm
+    from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
+    from litellm.types.router import LiteLLM_Params
+
+    monkeypatch.setattr(litellm, "check_provider_endpoint", True)
+    api_base: Final = "https://custom-openai-no-v1.test/serving"
+    respx_mock.get(f"{api_base}/models").mock(
+        return_value=httpx.Response(200, json={"data": [{"id": model_id} for model_id in _SERVER_MODEL_IDS]})
+    )
+
+    result: Final = get_known_models_from_wildcard(
+        wildcard_model="my-server/*",
+        litellm_params=LiteLLM_Params(model="custom_openai/*", api_base=api_base, api_key="sk-custom"),
+    )
+
+    assert result == [f"my-server/{model_id}" for model_id in _SERVER_MODEL_IDS]
+
+
+def test_openai_compatible_server_wildcard_lists_nothing_when_the_server_rejects_the_listing(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    import litellm
+    from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
+    from litellm.types.router import LiteLLM_Params
+
+    monkeypatch.setattr(litellm, "check_provider_endpoint", True)
+    api_base: Final = "https://custom-openai-rejects.test/v1"
+    with respx.mock(assert_all_called=True) as respx_mock:
+        respx_mock.get(f"{api_base}/models").mock(return_value=httpx.Response(401, json={"error": "bad key"}))
+        result: Final = get_known_models_from_wildcard(
+            wildcard_model="my-server/*",
+            litellm_params=LiteLLM_Params(model="custom_openai/*", api_base=api_base, api_key="sk-wrong"),
+        )
+
+    assert result == []