From 9e0d891335f575f980c5dc8b90dab8b2bda3ae39 Mon Sep 17 00:00:00 2001 From: Aniket Wagh Date: Fri, 28 Aug 2026 00:58:54 +0530 Subject: [PATCH] fix(proxy): expand wildcards for providers discovered from their endpoint get_provider_models gated provider-endpoint discovery on membership in the static litellm.models_by_provider catalog. Providers whose model list is dynamic, litellm_proxy and hosted_vllm, have no static catalog and so are absent from that dict, which meant discovery was skipped for exactly the providers that depend on it. The effect was that a chained proxy, LiteLLM in front of LiteLLM, answered GET /v1/models with the literal string litellm_proxy/* instead of the upstream's model list, and never sent a request upstream. Routing was unaffected, so only discovery was broken. Admit a provider when it registers a model-info config and check_provider_endpoint is set, which is the same pair of conditions get_valid_models already applies before fetching. Discovery stays opt-in and providers with a static catalog are untouched. Fixes #38547 --- litellm/proxy/auth/model_checks.py | 24 +++++- .../proxy/auth/test_model_checks.py | 77 ++++++++++++++++++- 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 1625198892f..c6253aaaf17 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -13,7 +13,7 @@ from litellm.router import Router from litellm.router_utils.fallback_event_handlers import get_fallback_model_group from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders -from litellm.utils import get_valid_models +from litellm.utils import ProviderConfigManager, get_valid_models _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) @@ -35,6 +35,26 @@ def _check_wildcard_routing(model: str) -> bool: return False +def _can_discover_models_from_provider_endpoint(provider: str) -> bool: + """ + True if `provider` can list its models by querying its own endpoint. + + Dynamic-list providers such as `litellm_proxy` and `hosted_vllm` have no static + catalog, so gating discovery on `litellm.models_by_provider` alone skips exactly the + providers that need it. Mirrors the condition `get_valid_models` applies before + fetching, so the gate only opens where discovery would actually happen + """ + if not litellm.check_provider_endpoint: + return False + + try: + provider_enum: Final = LlmProviders(provider) + except ValueError: + return False + + return ProviderConfigManager.get_provider_model_info(model=None, provider=provider_enum) is not None + + def get_provider_models(provider: str, litellm_params: LiteLLM_Params | None = None) -> list[str] | None: """ Returns the list of known models by provider @@ -42,7 +62,7 @@ def get_provider_models(provider: str, litellm_params: LiteLLM_Params | None = N if provider == "*": return get_valid_models(litellm_params=litellm_params) - if provider in litellm.models_by_provider: + if provider in litellm.models_by_provider or _can_discover_models_from_provider_endpoint(provider): provider_models: Final = get_valid_models(custom_llm_provider=provider, litellm_params=litellm_params) return provider_models return None diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 62073f4bf51..747be28e788 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -1,9 +1,12 @@ -from unittest.mock import AsyncMock, patch +from typing import Final +from unittest.mock import AsyncMock, Mock, patch import pytest +import litellm from litellm.proxy._types import LiteLLM_TeamTable, LiteLLM_UserTable, Member from litellm.proxy.auth.handle_jwt import JWTAuthManager +from litellm.types.router import LiteLLM_Params def test_get_team_models_for_all_models_and_team_only_models(): @@ -803,3 +806,75 @@ def test_get_complete_model_list_sentinel_only_grants_nothing(): infer_model_from_keys=False, ) assert result == [] + +class TestProviderEndpointModelDiscovery: + """`get_provider_models` gated provider-endpoint discovery on membership in the + static `litellm.models_by_provider` catalog. Dynamic-list providers like + `litellm_proxy` and `hosted_vllm` are absent from that dict, so discovery was + skipped for exactly the providers needing it: a chained proxy's GET /v1/models + returned the literal `litellm_proxy/*` and never queried the upstream (#38547)""" + + @staticmethod + def _upstream_params() -> LiteLLM_Params: + return LiteLLM_Params( + model="litellm_proxy/*", + api_base="http://127.0.0.1:4010", + api_key="sk-upstream-1234", + ) + + def test_dynamic_provider_delegates_to_discovery_when_enabled(self, monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy.auth import model_checks + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + discover: Final = Mock(return_value=["gpt-4o", "claude-sonnet"]) + monkeypatch.setattr(model_checks, "get_valid_models", discover) + + params: Final = self._upstream_params() + result: Final = model_checks.get_provider_models("litellm_proxy", params) + + assert result == ["gpt-4o", "claude-sonnet"] + discover.assert_called_once_with(custom_llm_provider="litellm_proxy", litellm_params=params) + + def test_dynamic_provider_returns_none_when_discovery_disabled(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Discovery stays opt-in, so nothing should be fetched with the flag off""" + from litellm.proxy.auth import model_checks + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + discover: Final = Mock(return_value=["unused"]) + monkeypatch.setattr(model_checks, "get_valid_models", discover) + + assert model_checks.get_provider_models("litellm_proxy", self._upstream_params()) is None + discover.assert_not_called() + + def test_unknown_provider_returns_none(self, monkeypatch: pytest.MonkeyPatch) -> None: + """An unrecognised provider must not raise out of the gate's enum conversion""" + from litellm.proxy.auth.model_checks import get_provider_models + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + + assert get_provider_models("not_a_real_provider") is None + + @pytest.mark.parametrize("provider", ["litellm_proxy", "hosted_vllm"]) + def test_gate_admits_dynamic_providers_when_enabled(self, monkeypatch: pytest.MonkeyPatch, provider: str) -> None: + """The gate is not specific to litellm_proxy: it covers every provider whose + model list comes from its endpoint rather than a static catalog""" + from litellm.proxy.auth.model_checks import _can_discover_models_from_provider_endpoint + + assert provider not in litellm.models_by_provider + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + assert _can_discover_models_from_provider_endpoint(provider) is True + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + assert _can_discover_models_from_provider_endpoint(provider) is False + + def test_static_catalog_provider_unaffected(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Providers with a static catalog keep listing from it with discovery off""" + from litellm.proxy.auth.model_checks import get_provider_models + + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + + result: Final = get_provider_models("anthropic") + + assert result is not None + assert any("claude" in model for model in result)