From d90200003b22358645308f9a314740cc02a497f5 Mon Sep 17 00:00:00 2001 From: jesus Date: Sun, 6 Sep 2026 18:53:32 +0000 Subject: [PATCH 01/14] fix(proxy): expand wildcard models for providers with model info but no cost map entry Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/model_checks.py | 14 +++++- .../proxy/auth/test_model_checks.py | 43 ++++++++++++++++++- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index de2ca4762f1..55b4670d14e 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -13,7 +13,7 @@ from litellm.router import Router from litellm.router_utils.fallback_event_handlers import get_fallback_model_group from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders -from litellm.utils import get_valid_models +from litellm.utils import ProviderConfigManager, get_valid_models _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) @@ -35,6 +35,16 @@ def _check_wildcard_routing(model: str) -> bool: return False +def _provider_supports_model_discovery(provider: str) -> bool: + if provider in litellm.models_by_provider: + return True + try: + llm_provider: Final = LlmProviders(provider) + except ValueError: + return False + return ProviderConfigManager.get_provider_model_info(model=None, provider=llm_provider) is not None + + def get_provider_models(provider: str, litellm_params: LiteLLM_Params | None = None) -> list[str] | None: """ Returns the list of known models by provider @@ -42,7 +52,7 @@ def get_provider_models(provider: str, litellm_params: LiteLLM_Params | None = N if provider == "*": return get_valid_models(litellm_params=litellm_params) - if provider in litellm.models_by_provider: + if _provider_supports_model_discovery(provider): provider_models: Final = get_valid_models(custom_llm_provider=provider, litellm_params=litellm_params) return provider_models return None diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 36bfc4c5dd3..f580df2dad6 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -1,4 +1,4 @@ -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -432,6 +432,47 @@ def test_wildcard_credential_hydration_preserves_deployment_params( } +def test_get_known_models_from_wildcard_hosted_vllm_uses_provider_endpoint(): + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + response = MagicMock() + response.json.return_value = { + "data": [ + {"id": "meta-llama/Llama-3.1-8B-Instruct"}, + {"id": "qwen2.5"}, + ] + } + original_check_provider_endpoint = litellm.check_provider_endpoint + try: + litellm.check_provider_endpoint = True # test-quality-ok: required to exercise provider endpoint discovery + with ( + patch("litellm.module_level_client.get", return_value=response), # test-quality-ok: required HTTP boundary + patch( # test-quality-ok: VLLM model listing has no injectable API key seam + "litellm.llms.vllm.common_utils.VLLMModelInfo.get_api_key", + return_value="test-key", + ), + ): + result = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params(model="hosted_vllm/*", api_base="http://localhost:8000/v1"), + ) + finally: + litellm.check_provider_endpoint = original_check_provider_endpoint # test-quality-ok: restore test global + + assert result == [ + "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct", + "hosted_vllm/qwen2.5", + ] + + +def test_get_known_models_from_wildcard_unknown_provider_returns_empty(): + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + assert get_known_models_from_wildcard("not_a_real_provider/*") == [] + + def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): """Regression test for #30358. From 2ed624550d872f043ceb22ebb14797e09f4ab94f Mon Sep 17 00:00:00 2001 From: jesus Date: Sun, 6 Sep 2026 18:57:51 +0000 Subject: [PATCH 02/14] fix(vllm): stop requiring an api key to list vllm models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/vllm/common_utils.py | 9 +-------- tests/test_litellm/proxy/auth/test_model_checks.py | 4 ---- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index 2da2269e1f9..f663ec7870a 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -47,9 +47,7 @@ class VLLMModelInfo(BaseLLMModelInfo): def get_api_base(api_base: str | None = None) -> str | None: api_base = api_base or get_secret_str("VLLM_API_BASE") if api_base is None: - raise ValueError( - "VLLM_API_BASE is not set. Please set the environment variable, to use VLLM's pass-through - `{LITELLM_API_BASE}/vllm/{endpoint}`." - ) + raise ValueError("VLLM_API_BASE is not set.") return api_base @staticmethod @@ -62,12 +60,7 @@ class VLLMModelInfo(BaseLLMModelInfo): def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: api_base = VLLMModelInfo.get_api_base(api_base) - api_key = VLLMModelInfo.get_api_key(api_key) endpoint: Final = "/v1/models" - if api_base is None or api_key is None: - raise ValueError( - "VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint." - ) url: Final = _add_path_to_api_base(api_base, endpoint) response: Final = litellm.module_level_client.get( diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index f580df2dad6..451f4eac9a0 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -449,10 +449,6 @@ def test_get_known_models_from_wildcard_hosted_vllm_uses_provider_endpoint(): litellm.check_provider_endpoint = True # test-quality-ok: required to exercise provider endpoint discovery with ( patch("litellm.module_level_client.get", return_value=response), # test-quality-ok: required HTTP boundary - patch( # test-quality-ok: VLLM model listing has no injectable API key seam - "litellm.llms.vllm.common_utils.VLLMModelInfo.get_api_key", - return_value="test-key", - ), ): result = get_known_models_from_wildcard( "hosted_vllm/*", From 624f4281aac239aa45ed291ed70de24a77ad3415 Mon Sep 17 00:00:00 2001 From: jesus Date: Sun, 6 Sep 2026 19:17:12 +0000 Subject: [PATCH 03/14] fix(proxy): preserve model ids during wildcard expansion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/auth/model_checks.py | 2 +- tests/test_litellm/proxy/auth/test_model_checks.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 55b4670d14e..fb925a3b133 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -325,7 +325,7 @@ def get_known_models_from_wildcard(wildcard_model: str, litellm_params: LiteLLM_ # Only strip the leading segment when it is a known provider, so ids whose first # segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it. leading, sep, model_suffix = model.partition("/") - if sep and leading in known_providers: + if sep and leading in known_providers and (provider in litellm.models_by_provider or leading == provider): model = f"{wildcard_provider_prefix}/{model_suffix}" else: model = f"{wildcard_provider_prefix}/{model}" diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 451f4eac9a0..bea46a3fd23 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -442,6 +442,7 @@ def test_get_known_models_from_wildcard_hosted_vllm_uses_provider_endpoint(): "data": [ {"id": "meta-llama/Llama-3.1-8B-Instruct"}, {"id": "qwen2.5"}, + {"id": "openai/foo"}, ] } original_check_provider_endpoint = litellm.check_provider_endpoint @@ -460,6 +461,7 @@ def test_get_known_models_from_wildcard_hosted_vllm_uses_provider_endpoint(): assert result == [ "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct", "hosted_vllm/qwen2.5", + "hosted_vllm/openai/foo", ] From 9edc1018fbe830d0e14e74e07b9aa0d27101aedc Mon Sep 17 00:00:00 2001 From: jesus Date: Sun, 6 Sep 2026 19:47:34 +0000 Subject: [PATCH 04/14] fix(vllm): forward api key for model discovery Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/vllm/common_utils.py | 9 ++++-- .../proxy/auth/test_model_checks.py | 28 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index f663ec7870a..106aae29aab 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -63,8 +63,13 @@ class VLLMModelInfo(BaseLLMModelInfo): endpoint: Final = "/v1/models" url: Final = _add_path_to_api_base(api_base, endpoint) - response: Final = litellm.module_level_client.get( - url=url, + response: Final = ( + litellm.module_level_client.get( + url=url, + headers={"x-api-key": api_key}, # mutable-ok: optional authentication header + ) + if api_key is not None + else litellm.module_level_client.get(url=url) ) response.raise_for_status() diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index bea46a3fd23..021d6a7f056 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -465,6 +465,34 @@ def test_get_known_models_from_wildcard_hosted_vllm_uses_provider_endpoint(): ] +def test_get_known_models_from_wildcard_hosted_vllm_forwards_api_key(): + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + response = MagicMock() + response.json.return_value = {"data": [{"id": "qwen2.5"}]} + original_check_provider_endpoint = litellm.check_provider_endpoint + try: + litellm.check_provider_endpoint = True # test-quality-ok: required to exercise provider endpoint discovery + with patch( # test-quality-ok: required HTTP boundary + "litellm.module_level_client.get", return_value=response + ) as mock_get: + result = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params( + model="hosted_vllm/*", + api_base="http://localhost:8000/v1", + api_key="test-key", + ), + ) + finally: + litellm.check_provider_endpoint = original_check_provider_endpoint # test-quality-ok: restore test global + + assert result == ["hosted_vllm/qwen2.5"] + assert mock_get.call_args.kwargs["headers"] == {"x-api-key": "test-key"} + + def test_get_known_models_from_wildcard_unknown_provider_returns_empty(): from litellm.proxy.auth.model_checks import get_known_models_from_wildcard From 519b979b23b23f428950ed98a0c97a97ee5d3ba9 Mon Sep 17 00:00:00 2001 From: jesus Date: Sun, 6 Sep 2026 20:15:07 +0000 Subject: [PATCH 05/14] fix(vllm): support bearer auth for model discovery Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/vllm/common_utils.py | 6 +++++- tests/test_litellm/proxy/auth/test_model_checks.py | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index 106aae29aab..f1ad1b0ce11 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -41,6 +41,7 @@ class VLLMModelInfo(BaseLLMModelInfo): ) -> dict: if api_key is not None: headers["x-api-key"] = api_key + headers["Authorization"] = f"Bearer {api_key}" return headers @staticmethod @@ -66,7 +67,10 @@ class VLLMModelInfo(BaseLLMModelInfo): response: Final = ( litellm.module_level_client.get( url=url, - headers={"x-api-key": api_key}, # mutable-ok: optional authentication header + headers={ # mutable-ok: optional authentication headers + "x-api-key": api_key, + "Authorization": f"Bearer {api_key}", + }, ) if api_key is not None else litellm.module_level_client.get(url=url) diff --git a/tests/test_litellm/proxy/auth/test_model_checks.py b/tests/test_litellm/proxy/auth/test_model_checks.py index 021d6a7f056..83e6e46ea1a 100644 --- a/tests/test_litellm/proxy/auth/test_model_checks.py +++ b/tests/test_litellm/proxy/auth/test_model_checks.py @@ -490,7 +490,10 @@ def test_get_known_models_from_wildcard_hosted_vllm_forwards_api_key(): litellm.check_provider_endpoint = original_check_provider_endpoint # test-quality-ok: restore test global assert result == ["hosted_vllm/qwen2.5"] - assert mock_get.call_args.kwargs["headers"] == {"x-api-key": "test-key"} + assert mock_get.call_args.kwargs["headers"] == { + "x-api-key": "test-key", + "Authorization": "Bearer test-key", + } def test_get_known_models_from_wildcard_unknown_provider_returns_empty(): From 1009886009b3944c37329fac210dfd815c611f62 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:39:58 -0700 Subject: [PATCH 06/14] fix(vllm): list hosted_vllm models from the server chat uses, with a bounded wait The hosted_vllm listing now resolves HOSTED_VLLM_API_BASE and HOSTED_VLLM_API_KEY like hosted_vllm chat, instead of the /vllm pass-through's VLLM_API_BASE, and waits at most NON_LLM_CONNECTION_TIMEOUT rather than the completion timeout. vllm/* keeps listing nothing, since its chat runs in process. The router pass-through now forwards the deployment's api_key. --- litellm/llms/vllm/common_utils.py | 56 +++++++++++++++------- litellm/utils.py | 10 ++-- tests/unit/passthrough/test_main.py | 21 ++++++++ tests/unit/proxy/auth/test_model_checks.py | 48 +++++++++++++++++++ 4 files changed, 113 insertions(+), 22 deletions(-) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index 3bd83387db2..edb3779a159 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -1,8 +1,10 @@ +from collections.abc import Mapping from typing import Final import httpx import litellm +from litellm.constants import NON_LLM_CONNECTION_TIMEOUT from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str @@ -28,6 +30,12 @@ class VLLMError(BaseLLMException): ) +def _auth_headers(api_key: str | None) -> Mapping[str, str]: + if api_key is None: + return {} + return {"x-api-key": api_key, "Authorization": f"Bearer {api_key}"} + + class VLLMModelInfo(BaseLLMModelInfo): def validate_environment( self, @@ -39,39 +47,51 @@ class VLLMModelInfo(BaseLLMModelInfo): api_key: str | None = None, api_base: str | None = None, ) -> dict: - if api_key is not None: - headers["x-api-key"] = api_key - headers["Authorization"] = f"Bearer {api_key}" + headers.update(_auth_headers(api_key)) return headers @staticmethod def get_api_base(api_base: str | None = None) -> str: api_base = api_base or get_secret_str("VLLM_API_BASE") if api_base is None: - raise ValueError("VLLM_API_BASE is not set.") + raise ValueError( + "VLLM_API_BASE is not set. Please set the environment variable, to use VLLM's pass-through - `{LITELLM_API_BASE}/vllm/{endpoint}`." + ) return api_base @staticmethod def get_api_key(api_key: str | None = None) -> str | None: - return None + return api_key @staticmethod def get_base_model(model: str) -> str | None: return model - def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: - api_base = VLLMModelInfo.get_api_base(api_base) - endpoint: Final = "/v1/models" - - url: Final = add_path_to_api_base(api_base, endpoint) - headers: Final = {"x-api-key": api_key, "Authorization": f"Bearer {api_key}"} if api_key is not None else None - response: Final = litellm.module_level_client.get(url=url, headers=headers) - - response.raise_for_status() - - models: Final = response.json()["data"] - - return [model["id"] for model in models] + def get_models( + self, api_key: str | None = None, api_base: str | None = None + ) -> list[str]: # mutable-ok: mirrors override contract + return [] def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return VLLMError(status_code=status_code, message=error_message, headers=headers) + + +class HostedVLLMModelInfo(VLLMModelInfo): + @staticmethod + def get_api_base(api_base: str | None = None) -> str: + resolved_api_base: Final = api_base or get_secret_str("HOSTED_VLLM_API_BASE") + if resolved_api_base is None: + raise ValueError("HOSTED_VLLM_API_BASE is not set.") + return resolved_api_base + + @staticmethod + def get_api_key(api_key: str | None = None) -> str | None: + return api_key or get_secret_str("HOSTED_VLLM_API_KEY") + + def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + url: Final = add_path_to_api_base(self.get_api_base(api_base), "/v1/models") + response: Final = litellm.module_level_client.get( + url=url, headers=dict(_auth_headers(self.get_api_key(api_key))), timeout=NON_LLM_CONNECTION_TIMEOUT + ) + response.raise_for_status() + return [model["id"] for model in response.json()["data"]] diff --git a/litellm/utils.py b/litellm/utils.py index 1e095966a43..48c50969058 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10049,12 +10049,14 @@ class ProviderConfigManager: from litellm.llms.ollama.common_utils import OllamaModelInfo return OllamaModelInfo() - elif LlmProviders.VLLM == provider or LlmProviders.HOSTED_VLLM == provider: - from litellm.llms.vllm.common_utils import ( - VLLMModelInfo, # experimental approach, to reduce bloat on __init__.py - ) + elif LlmProviders.VLLM == provider: + from litellm.llms.vllm.common_utils import VLLMModelInfo return VLLMModelInfo() + elif LlmProviders.HOSTED_VLLM == provider: + from litellm.llms.vllm.common_utils import HostedVLLMModelInfo + + return HostedVLLMModelInfo() elif LlmProviders.LEMONADE == provider: return litellm.LemonadeChatConfig() elif LlmProviders.CLARIFAI == provider: diff --git a/tests/unit/passthrough/test_main.py b/tests/unit/passthrough/test_main.py index 78fa83d126b..9c07729da86 100644 --- a/tests/unit/passthrough/test_main.py +++ b/tests/unit/passthrough/test_main.py @@ -51,3 +51,24 @@ async def test_allm_passthrough_route_hosted_vllm_sends_normalized_model(respx_m outbound: Final = json.loads(route.calls[0].request.content) assert outbound["model"] == "my-deployment" assert outbound["messages"] == [{"role": "user", "content": "Hello"}] + + +@pytest.mark.asyncio +async def test_allm_passthrough_route_hosted_vllm_sends_the_deployment_api_key(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + response: Final = await allm_passthrough_route( + method="POST", + endpoint="v1/chat/completions", + model="hosted_vllm/my-deployment", + api_base=_API_BASE, + api_key="deployment-key", + json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, + client=AsyncHTTPHandler(), + ) + assert response.status_code == 200 + sent_headers: Final = route.calls[0].request.headers + assert sent_headers["authorization"] == "Bearer deployment-key" + assert sent_headers["x-api-key"] == "deployment-key" + diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 490222361d2..623eed1ba7b 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -481,6 +481,54 @@ def test_hosted_vllm_wildcard_sends_the_configured_api_key_as_a_bearer_token(mon assert models_route.call_count == 1 +@respx.mock +def test_hosted_vllm_wildcard_without_api_base_lists_the_server_chat_would_call(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setenv("HOSTED_VLLM_API_BASE", "http://hosted-vllm-env.test:8000/v1") + monkeypatch.setenv("HOSTED_VLLM_API_KEY", "hosted-vllm-env-key") + monkeypatch.setenv("VLLM_API_BASE", "http://vllm-passthrough-env.test:8000") + passthrough_server_route: Final = respx.get("http://vllm-passthrough-env.test:8000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "model-on-the-passthrough-server"}]}) + ) + respx.get( + "http://hosted-vllm-env.test:8000/v1/models", headers={"Authorization": "Bearer hosted-vllm-env-key"} + ).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + + listed: Final = get_known_models_from_wildcard( + "hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", custom_llm_provider="hosted_vllm") + ) + + assert listed == ["hosted_vllm/qwen2.5"] + assert not passthrough_server_route.called + + +@respx.mock +def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_hung_server( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.constants import NON_LLM_CONNECTION_TIMEOUT + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + models_route: Final = respx.get("http://vllm-timeout.test:8000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + + get_known_models_from_wildcard( + "hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-timeout.test:8000/v1") + ) + + read_timeout: Final = models_route.calls.last.request.extensions["timeout"]["read"] + assert read_timeout is not None + assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT + + def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): """Regression test for #30358. From 4e109a03e910479a07bf5b2873a7440a8d160c70 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:07:35 -0700 Subject: [PATCH 07/14] fix(vllm): skip the placeholder key, honor deployment TLS and headers, and retry failed listings after a minute --- litellm/constants.py | 1 + litellm/llms/vllm/common_utils.py | 43 +++++- litellm/proxy/auth/model_checks.py | 43 +++--- litellm/utils.py | 10 +- .../llms/custom_httpx/test_http_handler.py | 7 +- tests/unit/passthrough/test_main.py | 30 ++++ tests/unit/proxy/auth/test_model_checks.py | 143 +++++++++++++++++- tests/unit/test_utils.py | 29 ++++ 8 files changed, 272 insertions(+), 34 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 8cc51d0ab5e..e458144474b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -610,6 +610,7 @@ DEFAULT_REDIS_MAJOR_VERSION: Final = get_env_int("DEFAULT_REDIS_MAJOR_VERSION", NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int( "NON_LLM_CONNECTION_TIMEOUT", 15 ) # timeout for adjacent services (e.g. jwt auth) +PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS: Final = 60 MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000) MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048) MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024 diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index edb3779a159..2340c6ba133 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -2,11 +2,12 @@ from collections.abc import Mapping from typing import Final import httpx +from pydantic import TypeAdapter -import litellm -from litellm.constants import NON_LLM_CONNECTION_TIMEOUT +from litellm.constants import NON_LLM_CONNECTION_TIMEOUT, PLACEHOLDER_API_KEY from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.custom_httpx.http_handler import get_httpx_client from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.utils import add_path_to_api_base @@ -30,12 +31,20 @@ class VLLMError(BaseLLMException): ) +_EXTRA_HEADERS: Final = TypeAdapter(Mapping[str, str]) + + def _auth_headers(api_key: str | None) -> Mapping[str, str]: - if api_key is None: + if api_key is None or api_key == PLACEHOLDER_API_KEY: return {} return {"x-api-key": api_key, "Authorization": f"Bearer {api_key}"} +def _str_param(litellm_params: Mapping[str, object], key: str) -> str | None: + value: Final = litellm_params.get(key) + return value if isinstance(value, str) else None + + class VLLMModelInfo(BaseLLMModelInfo): def validate_environment( self, @@ -89,9 +98,33 @@ class HostedVLLMModelInfo(VLLMModelInfo): return api_key or get_secret_str("HOSTED_VLLM_API_KEY") def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]: + return self._list_models(api_key=api_key, api_base=api_base, extra_headers={}, ssl_verify=None) + + def discover_models( + self, litellm_params: Mapping[str, object] | None = None + ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override + params: Final = litellm_params or {} + ssl_verify: Final = params.get("ssl_verify") + return self._list_models( + api_key=_str_param(params, "api_key"), + api_base=_str_param(params, "api_base"), + extra_headers=_EXTRA_HEADERS.validate_python(params.get("extra_headers") or {}), + ssl_verify=ssl_verify if isinstance(ssl_verify, (bool, str)) else None, + ) + + def _list_models( + self, + *, + api_key: str | None, + api_base: str | None, + extra_headers: Mapping[str, str], + ssl_verify: bool | str | None, + ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override url: Final = add_path_to_api_base(self.get_api_base(api_base), "/v1/models") - response: Final = litellm.module_level_client.get( - url=url, headers=dict(_auth_headers(self.get_api_key(api_key))), timeout=NON_LLM_CONNECTION_TIMEOUT + response: Final = get_httpx_client(params={"ssl_verify": ssl_verify}).get( + url=url, + headers={**extra_headers, **_auth_headers(self.get_api_key(api_key))}, + timeout=NON_LLM_CONNECTION_TIMEOUT, ) response.raise_for_status() return [model["id"] for model in response.json()["data"]] diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 5722bf7347e..1ddfc5bd805 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -284,12 +284,14 @@ _KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders) def _prefix_wildcard_model(model: str, wildcard_provider_prefix: str, provider: str) -> str: if model.startswith(f"{wildcard_provider_prefix}/"): return model - # `get_provider_models` returns provider-prefixed ids (e.g. "ollama/gemma3:1b"). + # `get_provider_models` can return provider-prefixed ids (e.g. "ollama/gemma3:1b"). # When the wildcard uses a custom prefix (e.g. "ollama_server1/*" to distinguish # multiple instances), replace that existing provider prefix instead of stacking # both, which would otherwise yield an uncallable "ollama_server1/ollama/gemma3:1b". - # Only strip the leading segment when it is a known provider, so ids whose first - # segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it. + # Only strip the leading segment when it is a known provider that prefixes this + # provider's ids: either the provider's own name or, for a provider with a static + # catalog, any provider. Ids a server reports with an org or another provider's + # name first (e.g. "meta-llama/Llama-3-8B" or "openai/gpt-oss-20b" on vLLM) keep it. leading, sep, model_suffix = model.partition("/") if sep and leading in _KNOWN_PROVIDERS and (_has_static_catalog(provider) or leading == provider): return f"{wildcard_provider_prefix}/{model_suffix}" @@ -548,30 +550,27 @@ def _get_wildcard_models( llm_router: Router | None = None, team_id: str | None = None, ) -> list[str]: - models_to_remove: Final = set() + wildcard_routes: Final = tuple(model for model in unique_models if _check_wildcard_routing(model=model)) routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) all_wildcard_models: Final = [] - for model in unique_models: - if _check_wildcard_routing(model=model): - if return_wildcard_routes: - all_wildcard_models.append(model) + for model in wildcard_routes: + if return_wildcard_routes: + all_wildcard_models.append(model) - models_to_remove.add(model) - - model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None - if model_list: - for router_model in model_list: - all_wildcard_models.extend( - get_known_models_from_wildcard( - wildcard_model=model, - litellm_params=LiteLLM_Params(**router_model["litellm_params"]), - routes_to_model=routes_to_model, - ) + model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None + if model_list: + for router_model in model_list: + all_wildcard_models.extend( + get_known_models_from_wildcard( + wildcard_model=model, + litellm_params=LiteLLM_Params(**router_model["litellm_params"]), + routes_to_model=routes_to_model, ) - else: - all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None)) + ) + else: + all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None)) - for model in models_to_remove: + for model in frozenset(wildcard_routes): unique_models.remove(model) return all_wildcard_models diff --git a/litellm/utils.py b/litellm/utils.py index 48c50969058..50c3d17290e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -28,7 +28,7 @@ import textwrap import threading import time import traceback -from contextlib import nullcontext +from contextlib import nullcontext, suppress from dataclasses import dataclass, field from functools import lru_cache, wraps from importlib import resources @@ -79,6 +79,7 @@ from litellm.constants import ( MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE, NON_INFERENCE_CALL_TYPES, OPENAI_EMBEDDING_PARAMS, + PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS, PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) @@ -8386,10 +8387,11 @@ class AvailableModelsCache(InMemoryCache): custom_llm_provider: str, litellm_params: LiteLLM_Params | None, available_models: list[str], + ttl: int | None = None, ): """Set cached model info""" cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) - self.set_cache(cache_key, copy.deepcopy(available_models)) + self.set_cache(cache_key, copy.deepcopy(available_models), ttl=ttl) # Global cache instance @@ -8455,6 +8457,10 @@ def _get_valid_models_from_provider_api( return models except Exception as e: verbose_logger.warning("Error getting valid models: %s", e) + with suppress(ValueError): + _model_cache.set_cached_model_info( + custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + ) return [] diff --git a/tests/unit/llms/custom_httpx/test_http_handler.py b/tests/unit/llms/custom_httpx/test_http_handler.py index 1607447f7bf..f5a7de1b2ed 100644 --- a/tests/unit/llms/custom_httpx/test_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_http_handler.py @@ -291,13 +291,12 @@ async def test_ssl_context_with_shared_session(monkeypatch: pytest.MonkeyPatch): await shared_session.close() -def test_get_ssl_configuration(): +def test_get_ssl_configuration(monkeypatch): """Test that get_ssl_configuration() returns a proper SSL context with certifi CA bundle when no environment variables are set.""" - from litellm.llms.custom_httpx.http_handler import _ssl_context_cache + from litellm.llms.custom_httpx import http_handler - # Clear cache to ensure ssl.create_default_context is called - _ssl_context_cache.clear() + monkeypatch.setattr(http_handler, "_ssl_context_cache", {}) with patch.dict(os.environ, clear=True): with patch("ssl.create_default_context") as mock_create_context: diff --git a/tests/unit/passthrough/test_main.py b/tests/unit/passthrough/test_main.py index 9c07729da86..e5369e44068 100644 --- a/tests/unit/passthrough/test_main.py +++ b/tests/unit/passthrough/test_main.py @@ -72,3 +72,33 @@ async def test_allm_passthrough_route_hosted_vllm_sends_the_deployment_api_key(r assert sent_headers["authorization"] == "Bearer deployment-key" assert sent_headers["x-api-key"] == "deployment-key" + + +@pytest.mark.asyncio +async def test_router_passthrough_to_a_hosted_vllm_deployment_without_an_api_key_sends_no_auth_headers( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("HOSTED_VLLM_API_KEY", raising=False) + route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "vllm-without-key", + "litellm_params": {"model": "hosted_vllm/my-deployment", "api_base": _API_BASE}, + } + ] + ) + + response: Final = await router.allm_passthrough_route( + model="vllm-without-key", + method="POST", + endpoint="v1/chat/completions", + json={"model": "anything", "messages": [{"role": "user", "content": "Hello"}]}, + ) + + assert response.status_code == 200 + sent_headers: Final = route.calls[0].request.headers + assert "authorization" not in sent_headers + assert "x-api-key" not in sent_headers diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 623eed1ba7b..8415f593213 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1,11 +1,22 @@ -from collections.abc import Generator, Mapping +import datetime +import ipaddress +import json +import ssl +import threading +from collections.abc import Generator, Iterator, Mapping from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from typing import Final from unittest.mock import patch import httpx import pytest import respx +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509.oid import NameOID def test_get_team_models_for_all_models_and_team_only_models(): @@ -529,6 +540,136 @@ def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_ assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT +@respx.mock +def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + respx.get( + "http://vllm-behind-gateway.test:8000/v1/models", + headers={"X-Tenant": "team-a", "Authorization": "Bearer vllm-test-key"}, + ).mock(return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + + listed: Final = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params( + model="hosted_vllm/*", + api_base="http://vllm-behind-gateway.test:8000/v1", + api_key="vllm-test-key", + extra_headers={"X-Tenant": "team-a"}, + ), + ) + + assert listed == ["hosted_vllm/qwen2.5"] + + +def _self_signed_cert(directory: Path) -> tuple[Path, Path]: + key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")]) + now: Final = datetime.datetime.now(datetime.UTC) + cert: Final = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=1)) + .add_extension( + x509.SubjectAlternativeName([x509.IPAddress(ipaddress.IPv4Address("127.0.0.1"))]), critical=False + ) + .sign(key, hashes.SHA256()) + ) + cert_path: Final = directory / "cert.pem" + key_path: Final = directory / "key.pem" + cert_path.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_path.write_bytes( + key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()) + ) + return cert_path, key_path + + +@pytest.fixture +def self_signed_vllm_api_base(tmp_path: Path) -> Iterator[str]: + cert_path, key_path = _self_signed_cert(tmp_path) + + class Handler(BaseHTTPRequestHandler): + timeout = 5 + + def do_GET(self) -> None: + body: Final = json.dumps({"data": [{"id": "qwen2.5"}]}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: # matches BaseHTTPRequestHandler + pass + + server: Final = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + context: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert_path, key_path) + server.socket = context.wrap_socket(server.socket, server_side=True, do_handshake_on_connect=False) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield f"https://127.0.0.1:{server.server_address[1]}/v1" + finally: + server.shutdown() + server.server_close() + + +def test_hosted_vllm_wildcard_lists_a_self_signed_server_only_when_the_deployment_turns_off_ssl_verify( + monkeypatch: pytest.MonkeyPatch, self_signed_vllm_api_base: str +) -> None: + import litellm + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.llms.custom_httpx import http_handler + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + monkeypatch.setattr(litellm, "ssl_verify", True) + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + monkeypatch.setattr(http_handler, "_ssl_context_cache", {}) + monkeypatch.delenv("SSL_VERIFY", raising=False) + monkeypatch.delenv("SSL_CERT_FILE", raising=False) + + verified: Final = get_known_models_from_wildcard( + "hosted_vllm/*", LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base) + ) + unverified: Final = get_known_models_from_wildcard( + "hosted_vllm/*", + LiteLLM_Params(model="hosted_vllm/*", api_base=self_signed_vllm_api_base, ssl_verify=False), + ) + + assert verified == [] + assert unverified == ["hosted_vllm/qwen2.5"] + + +@respx.mock +def test_custom_prefix_over_a_litellm_proxy_wildcard_lists_upstream_models_under_the_prefix_alone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + from litellm.types.router import LiteLLM_Params + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + respx.get("http://upstream-proxy.test:4000/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "gpt-4o"}, {"id": "claude-sonnet"}]}) + ) + + listed: Final = get_known_models_from_wildcard( + "team-proxy/*", + LiteLLM_Params(model="litellm_proxy/*", api_base="http://upstream-proxy.test:4000", api_key="sk-upstream"), + ) + + assert listed == ["team-proxy/gpt-4o", "team-proxy/claude-sonnet"] + + def test_wildcard_custom_prefix_does_not_stack_provider_prefix(monkeypatch): """Regression test for #30358. diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 38287b25169..7511c7d8b7a 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10581,6 +10581,35 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch): assert _model_cache.get_cached_model_info("openai") is None +@respx.mock +def test_a_failed_provider_model_listing_is_retried_only_after_its_failure_ttl(monkeypatch: pytest.MonkeyPatch) -> None: + import time + + from litellm.constants import PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + from litellm.types.router import LiteLLM_Params + + models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock( + side_effect=httpx.ReadTimeout("vLLM did not answer") + ) + litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-hung.test:8000/v1") + failed_at: Final = time.time() + + def list_models() -> list[str]: + return get_valid_models( + check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=litellm_params + ) + + assert list_models() == [] + assert list_models() == [] + assert models_route.call_count == 1 + + models_route.mock(side_effect=None, return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + 1) + + assert list_models() == ["qwen2.5"] + assert models_route.call_count == 2 + + def test_delta_tool_calls_sequential_indices(): """ Test that multiple tool calls without explicit indices receive sequential indices. From 7a575fea15f0acaf2e59eeefd73107be6c91598c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:39:53 -0700 Subject: [PATCH 08/14] fix(discovery): cache only timed-out model listings, so a fast failure is tried again on the next call --- litellm/constants.py | 2 +- litellm/utils.py | 11 +++++++---- tests/unit/test_utils.py | 34 +++++++++++++++++++++++++++++++--- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index e458144474b..dc8107f2343 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -610,7 +610,7 @@ DEFAULT_REDIS_MAJOR_VERSION: Final = get_env_int("DEFAULT_REDIS_MAJOR_VERSION", NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int( "NON_LLM_CONNECTION_TIMEOUT", 15 ) # timeout for adjacent services (e.g. jwt auth) -PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS: Final = 60 +PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS: Final = 60 MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000) MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048) MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024 diff --git a/litellm/utils.py b/litellm/utils.py index 50c3d17290e..d89f405223e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -79,7 +79,7 @@ from litellm.constants import ( MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE, NON_INFERENCE_CALL_TYPES, OPENAI_EMBEDDING_PARAMS, - PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS, + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS, PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) @@ -8455,13 +8455,16 @@ def _get_valid_models_from_provider_api( _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) return models - except Exception as e: - verbose_logger.warning("Error getting valid models: %s", e) + except httpx.TimeoutException as e: + verbose_logger.warning("Timed out getting valid models: %s", e) with suppress(ValueError): _model_cache.set_cached_model_info( - custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS ) return [] + except Exception as e: + verbose_logger.warning("Error getting valid models: %s", e) + return [] def get_valid_models( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7511c7d8b7a..04e4d7e2b51 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10582,10 +10582,12 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch): @respx.mock -def test_a_failed_provider_model_listing_is_retried_only_after_its_failure_ttl(monkeypatch: pytest.MonkeyPatch) -> None: +def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_ttl( + monkeypatch: pytest.MonkeyPatch, +) -> None: import time - from litellm.constants import PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + from litellm.constants import PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS from litellm.types.router import LiteLLM_Params models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock( @@ -10604,12 +10606,38 @@ def test_a_failed_provider_model_listing_is_retried_only_after_its_failure_ttl(m assert models_route.call_count == 1 models_route.mock(side_effect=None, return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) - monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_FAILURE_CACHE_TTL_SECONDS + 1) + monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + 1) assert list_models() == ["qwen2.5"] assert models_route.call_count == 2 + +@pytest.mark.parametrize( + ("host", "first_answer"), + [("vllm-erroring.test", httpx.Response(503)), ("vllm-refusing.test", httpx.ConnectError("connection refused"))], + ids=["server-error", "connection-refused"], +) +@respx.mock +def test_a_provider_model_listing_that_fails_fast_is_tried_again_on_the_next_call( + host: str, first_answer: httpx.Response | httpx.ConnectError +) -> None: + from litellm.types.router import LiteLLM_Params + + models_route: Final = respx.get(f"http://{host}:8000/v1/models").mock( + side_effect=[first_answer, httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})] + ) + litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base=f"http://{host}:8000/v1") + + def list_models() -> list[str]: + return get_valid_models( + check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=litellm_params + ) + + assert list_models() == [] + assert list_models() == ["qwen2.5"] + assert models_route.call_count == 2 + def test_delta_tool_calls_sequential_indices(): """ Test that multiple tool calls without explicit indices receive sequential indices. From d69da65a6aee365d6dfb587d76157f604ff5e4d8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:03:45 -0700 Subject: [PATCH 09/14] fix(proxy): list wildcard models off the event loop, so a hung provider no longer stalls other requests --- litellm/proxy/proxy_server.py | 8 ++-- litellm/proxy/utils.py | 6 ++- litellm/utils.py | 11 ++--- tests/unit/proxy/auth/test_model_checks.py | 48 ++++++++++++++++++++++ tests/unit/test_utils.py | 22 +++++----- 5 files changed, 75 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3b36ea5afce..84b05b88a41 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12398,7 +12398,8 @@ async def model_list( # Get complete model list including wildcard routes if requested from litellm.proxy.auth.model_checks import get_complete_model_list - all_models = get_complete_model_list( + all_models = await asyncio.to_thread( + get_complete_model_list, key_models=[], team_models=[], proxy_model_list=proxy_model_list, @@ -17751,7 +17752,7 @@ async def model_info_v1( alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias()) all_models.extend(alias_models) - all_models = expand_wildcard_deployments_for_model_info(all_models, llm_router) + all_models = await asyncio.to_thread(expand_wildcard_deployments_for_model_info, all_models, llm_router) allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=user_api_key_dict, @@ -18086,7 +18087,8 @@ async def model_group_info( LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, ) all_models_str: Final = ( - get_complete_model_list( + await asyncio.to_thread( + get_complete_model_list, key_models=(), team_models=(), proxy_model_list=llm_router.get_model_names(), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..cc3c7280160 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -9517,7 +9517,8 @@ async def _agent_access_group_visible_models( if not granted: return frozenset() return frozenset( - get_complete_model_list( + await asyncio.to_thread( + get_complete_model_list, key_models=granted, team_models=(), proxy_model_list=proxy_model_list, @@ -9652,7 +9653,8 @@ async def get_available_models_for_user( ) # Get complete model list - complete_models: Final = get_complete_model_list( + complete_models: Final = await asyncio.to_thread( + get_complete_model_list, key_models=granted_key_models, team_models=granted_team_models, proxy_model_list=proxy_model_list, diff --git a/litellm/utils.py b/litellm/utils.py index d89f405223e..71e3ff882e0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8333,8 +8333,8 @@ _HUGGINGFACE_TOKENIZER_LOAD_FAILURES: Final[_HuggingFaceTokenizerLoadFailureCach class AvailableModelsCache(InMemoryCache): - def __init__(self, ttl_seconds: int = 300, max_size: int = 1000): - super().__init__(ttl_seconds, max_size) + def __init__(self, ttl_seconds: int = 300, max_size: int = 1000, clock: Callable[[], float] | None = None): + super().__init__(ttl_seconds, max_size, clock=clock) self._env_hash: str | None = None def _get_env_hash(self) -> str: @@ -8443,9 +8443,10 @@ def _get_valid_models_from_provider_api( provider_config: BaseLLMModelInfo, custom_llm_provider: str, litellm_params: LiteLLM_Params | None = None, + model_cache: AvailableModelsCache = _model_cache, ) -> list[str]: try: - cached_result: Final = _model_cache.get_cached_model_info(custom_llm_provider, litellm_params) + cached_result: Final = model_cache.get_cached_model_info(custom_llm_provider, litellm_params) if cached_result is not None: return cached_result @@ -8453,12 +8454,12 @@ def _get_valid_models_from_provider_api( litellm_params=_provider_listing_params(custom_llm_provider, litellm_params) ) - _model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) + model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) return models except httpx.TimeoutException as e: verbose_logger.warning("Timed out getting valid models: %s", e) with suppress(ValueError): - _model_cache.set_cached_model_info( + model_cache.set_cached_model_info( custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS ) return [] diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 8415f593213..fa2f044f0e0 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -540,6 +540,54 @@ def test_hosted_vllm_wildcard_listing_does_not_wait_the_completion_timeout_on_a_ assert read_timeout <= NON_LLM_CONNECTION_TIMEOUT +@pytest.mark.asyncio +@respx.mock +async def test_a_hung_wildcard_listing_leaves_the_event_loop_free_for_other_requests( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import asyncio + + import litellm + from litellm import Router + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import get_available_models_for_user + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + listing_started: Final = threading.Event() + other_request_served: Final = threading.Event() + + def listing_that_answers_once_another_request_is_served(request: httpx.Request) -> httpx.Response: + listing_started.set() + if other_request_served.wait(timeout=3): + return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + return httpx.Response(503) + + respx.get("http://vllm-stalled.test:8000/v1/models").mock( + side_effect=listing_that_answers_once_another_request_is_served + ) + router: Final = Router( + model_list=[ + { + "model_name": "hosted_vllm/*", + "litellm_params": {"model": "hosted_vllm/*", "api_base": "http://vllm-stalled.test:8000/v1"}, + } + ] + ) + + listing: Final = asyncio.create_task( + get_available_models_for_user( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=[]), + llm_router=router, + general_settings={}, + user_model=None, + ) + ) + assert await asyncio.to_thread(listing_started.wait, 3) + other_request_served.set() + + assert "hosted_vllm/qwen2.5" in await listing + + @respx.mock def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None: import litellm diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 04e4d7e2b51..effc2d5e8a2 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10582,23 +10582,22 @@ def test_get_valid_models_from_provider_cache_invalidation(monkeypatch): @respx.mock -def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_ttl( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import time - +def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_ttl() -> None: from litellm.constants import PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + from litellm.llms.vllm.common_utils import HostedVLLMModelInfo from litellm.types.router import LiteLLM_Params + from litellm.utils import AvailableModelsCache, _get_valid_models_from_provider_api models_route: Final = respx.get("http://vllm-hung.test:8000/v1/models").mock( side_effect=httpx.ReadTimeout("vLLM did not answer") ) litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-hung.test:8000/v1") - failed_at: Final = time.time() + clock: Final = Mock(return_value=1000.0) + model_cache: Final = AvailableModelsCache(clock=clock) def list_models() -> list[str]: - return get_valid_models( - check_provider_endpoint=True, custom_llm_provider="hosted_vllm", litellm_params=litellm_params + return _get_valid_models_from_provider_api( + HostedVLLMModelInfo(), "hosted_vllm", litellm_params, model_cache=model_cache ) assert list_models() == [] @@ -10606,13 +10605,15 @@ def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_tt assert models_route.call_count == 1 models_route.mock(side_effect=None, return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) - monkeypatch.setattr(time, "time", lambda: failed_at + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + 1) + clock.return_value = 1000.0 + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS - 1 + assert list_models() == [] + assert models_route.call_count == 1 + clock.return_value = 1000.0 + PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + 1 assert list_models() == ["qwen2.5"] assert models_route.call_count == 2 - @pytest.mark.parametrize( ("host", "first_answer"), [("vllm-erroring.test", httpx.Response(503)), ("vllm-refusing.test", httpx.ConnectError("connection refused"))], @@ -10638,6 +10639,7 @@ def test_a_provider_model_listing_that_fails_fast_is_tried_again_on_the_next_cal assert list_models() == ["qwen2.5"] assert models_route.call_count == 2 + def test_delta_tool_calls_sequential_indices(): """ Test that multiple tool calls without explicit indices receive sequential indices. From 3f7aed1cb8b7b5afe842ee37cdba2b3eb84fba46 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:03:47 -0700 Subject: [PATCH 10/14] fix(utils): keep a concurrent call's model listing when another call times out --- litellm/utils.py | 35 +++++++++++++++++++++++++++-------- tests/unit/test_utils.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 71e3ff882e0..c1b35e1daf3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -28,7 +28,7 @@ import textwrap import threading import time import traceback -from contextlib import nullcontext, suppress +from contextlib import nullcontext from dataclasses import dataclass, field from functools import lru_cache, wraps from importlib import resources @@ -8336,6 +8336,7 @@ class AvailableModelsCache(InMemoryCache): def __init__(self, ttl_seconds: int = 300, max_size: int = 1000, clock: Callable[[], float] | None = None): super().__init__(ttl_seconds, max_size, clock=clock) self._env_hash: str | None = None + self._listing_lock = threading.Lock() def _get_env_hash(self) -> str: """Create a hash of relevant environment variables""" @@ -8387,11 +8388,30 @@ class AvailableModelsCache(InMemoryCache): custom_llm_provider: str, litellm_params: LiteLLM_Params | None, available_models: list[str], - ttl: int | None = None, ): """Set cached model info""" + with self._listing_lock: + self._store_model_info(custom_llm_provider, litellm_params, available_models, ttl=None) + + def cache_timed_out_listing_unless_listed( + self, + custom_llm_provider: str, + litellm_params: LiteLLM_Params | None, + ttl: int, + ) -> None: + with self._listing_lock: + if self.get_cached_model_info(custom_llm_provider, litellm_params) is None: + self._store_model_info(custom_llm_provider, litellm_params, (), ttl=ttl) + + def _store_model_info( + self, + custom_llm_provider: str, + litellm_params: LiteLLM_Params | None, + available_models: Sequence[str], + ttl: int | None, + ) -> None: cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) - self.set_cache(cache_key, copy.deepcopy(available_models), ttl=ttl) + self.set_cache(cache_key, list(available_models), ttl=ttl) # Global cache instance @@ -8458,11 +8478,10 @@ def _get_valid_models_from_provider_api( return models except httpx.TimeoutException as e: verbose_logger.warning("Timed out getting valid models: %s", e) - with suppress(ValueError): - model_cache.set_cached_model_info( - custom_llm_provider, litellm_params, [], ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS - ) - return [] + model_cache.cache_timed_out_listing_unless_listed( + custom_llm_provider, litellm_params, ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS + ) + return model_cache.get_cached_model_info(custom_llm_provider, litellm_params) or [] except Exception as e: verbose_logger.warning("Error getting valid models: %s", e) return [] diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index effc2d5e8a2..53136d8495d 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10614,6 +10614,35 @@ def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_tt assert models_route.call_count == 2 +@respx.mock +def test_a_listing_that_times_out_keeps_the_models_a_concurrent_listing_cached() -> None: + from litellm.llms.vllm.common_utils import HostedVLLMModelInfo + from litellm.types.router import LiteLLM_Params + from litellm.utils import AvailableModelsCache, _get_valid_models_from_provider_api + + litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-flaky.test:8000/v1") + model_cache: Final = AvailableModelsCache(clock=Mock(return_value=1000.0)) + + def list_models() -> list[str]: + return _get_valid_models_from_provider_api( + HostedVLLMModelInfo(), "hosted_vllm", litellm_params, model_cache=model_cache + ) + + def time_out_after_a_concurrent_listing_succeeds(request: httpx.Request) -> httpx.Response: + assert list_models() == ["qwen2.5"] + raise httpx.ReadTimeout("vLLM did not answer", request=request) + + models_route: Final = respx.get("http://vllm-flaky.test:8000/v1/models").mock( + side_effect=iter( + (time_out_after_a_concurrent_listing_succeeds, httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})) + ) + ) + + assert list_models() == ["qwen2.5"] + assert list_models() == ["qwen2.5"] + assert models_route.call_count == 2 + + @pytest.mark.parametrize( ("host", "first_answer"), [("vllm-erroring.test", httpx.Response(503)), ("vllm-refusing.test", httpx.ConnectError("connection refused"))], From 22e417363a58d4e8d2bf93dd4c74bdce060b2b3e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 05:57:09 -0700 Subject: [PATCH 11/14] fix(proxy): run model discovery on its own thread pool and keep credential lookups off the event loop --- litellm/constants.py | 1 + litellm/proxy/auth/model_checks.py | 26 +++++- litellm/proxy/batches_endpoints/endpoints.py | 8 +- litellm/proxy/hooks/batch_rate_limiter.py | 8 +- .../openai_files_endpoints/common_utils.py | 9 +- .../openai_files_endpoints/files_endpoints.py | 10 +- litellm/proxy/proxy_server.py | 7 +- litellm/proxy/utils.py | 7 +- tests/unit/proxy/auth/test_model_checks.py | 53 +++++++++++ .../proxy/hooks/test_batch_file_validation.py | 15 +-- .../test_files_common_utils.py | 93 +++++++++++++++---- 11 files changed, 190 insertions(+), 47 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index baa213170ba..d2ac3d62377 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -612,6 +612,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int( "NON_LLM_CONNECTION_TIMEOUT", 15 ) # timeout for adjacent services (e.g. jwt auth) PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS: Final = 60 +PROVIDER_MODEL_DISCOVERY_MAX_THREADS: Final = 4 MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000) MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048) MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024 diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index b74a4f586b0..e6f6d60855a 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -1,14 +1,18 @@ # What is this? ## Common checks for /v1/models and `/model/info` +import asyncio +import contextvars import copy import functools import itertools import re from collections.abc import Callable, Mapping, Sequence -from typing import Final +from concurrent.futures import ThreadPoolExecutor +from typing import Final, ParamSpec, TypeVar import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import PROVIDER_MODEL_DISCOVERY_MAX_THREADS from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth @@ -22,6 +26,26 @@ from litellm.utils import ProviderConfigManager, get_valid_models LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) +_P: Final = ParamSpec("_P") +_R: Final = TypeVar("_R") + +_MODEL_DISCOVERY_EXECUTOR: Final = ThreadPoolExecutor( + max_workers=PROVIDER_MODEL_DISCOVERY_MAX_THREADS, thread_name_prefix="litellm-model-discovery" +) + + +async def run_model_discovery( + discover: Callable[_P, _R], + /, + *args: _P.args, + **kwargs: _P.kwargs, # kwargs-ok: forwards the discovery function's own arguments +) -> _R: + context: Final = contextvars.copy_context() + return await asyncio.get_running_loop().run_in_executor( + _MODEL_DISCOVERY_EXECUTOR, lambda: context.run(discover, *args, **kwargs) + ) + + _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index 763737191ef..f650ac3e3c2 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -489,7 +489,7 @@ async def create_batch( verbose_proxy_logger.debug("Created batch using model: %s", model_param) else: # SCENARIO 3: Fallback to custom_llm_provider (uses env variables) - apply_team_provider_credentials( + await apply_team_provider_credentials( data=cast(dict, _create_batch_data), # cast-ok: TypedDict is a dict at runtime llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -772,7 +772,7 @@ async def retrieve_batch( or get_custom_llm_provider_from_request_query(request=request) ) custom_llm_provider: Final = requested_provider or "openai" - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -986,7 +986,7 @@ async def list_batches( or get_custom_llm_provider_from_request_query(request=request) or "openai" ) - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -1209,7 +1209,7 @@ async def cancel_batch( # Extract batch_id from data to avoid "multiple values for keyword argument" error # data was cast from CancelBatchRequest which already contains batch_id data.pop("batch_id", None) - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/hooks/batch_rate_limiter.py b/litellm/proxy/hooks/batch_rate_limiter.py index 94762232e0b..f86338208ad 100644 --- a/litellm/proxy/hooks/batch_rate_limiter.py +++ b/litellm/proxy/hooks/batch_rate_limiter.py @@ -527,7 +527,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): return True return False - def _resolve_batch_input_file_fetch_params( + async def _resolve_batch_input_file_fetch_params( self, file_id: str, custom_llm_provider: str, @@ -541,6 +541,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): without decoding them, ``afile_content`` is called with the encoded ID and the upstream provider returns 404. """ + from litellm.proxy.auth.model_checks import run_model_discovery from litellm.proxy.openai_files_endpoints.common_utils import ( decode_model_from_file_id, explicit_request_credentials, @@ -591,7 +592,8 @@ class _PROXY_BatchRateLimiter(CustomLogger): return file_id, fetch_kwargs team_credentials: Final = ( - get_team_provider_credentials( + await run_model_discovery( + get_team_provider_credentials, llm_router=llm_router, user_api_key_dict=user_api_key_dict, custom_llm_provider=custom_llm_provider, @@ -948,7 +950,7 @@ class _PROXY_BatchRateLimiter(CustomLogger): user_api_key_dict=user_api_key_dict, ) else: - provider_file_id, fetch_kwargs = self._resolve_batch_input_file_fetch_params( + provider_file_id, fetch_kwargs = await self._resolve_batch_input_file_fetch_params( file_id=file_id, custom_llm_provider=custom_llm_provider, data=data or {}, diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index f7289445d3a..907fd11347c 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -530,7 +530,7 @@ def get_team_provider_credentials( custom_llm_provider: str, *, include_accessible_deployments: bool = True, -) -> dict | None: +) -> dict[str, object] | None: """ Resolve upstream credentials for a provider-scoped file operation (e.g. GET /v1/files), which doesn't pin a model. @@ -704,7 +704,7 @@ def explicit_request_credentials(data: Mapping[str, object]) -> Mapping[str, str return MappingProxyType({"api_key": api_key, "api_base": api_base}) -def apply_team_provider_credentials( +async def apply_team_provider_credentials( data: dict, # mutable-ok: credentials are merged into the request payload in place, same contract as prepare_data_with_credentials llm_router: Optional["Router"], user_api_key_dict: "UserAPIKeyAuth", @@ -719,7 +719,10 @@ def apply_team_provider_credentials( ``forward_llm_provider_auth_headers``) replaces the deployment's, and the request's ``api_base`` replaces the deployment's only alongside that key. """ - credentials: Final = get_team_provider_credentials( + from litellm.proxy.auth.model_checks import run_model_discovery + + credentials: Final = await run_model_discovery( + get_team_provider_credentials, llm_router=llm_router, user_api_key_dict=user_api_key_dict, custom_llm_provider=custom_llm_provider, diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 5d63c3db222..73f8fb470ec 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -524,7 +524,7 @@ async def route_create_file( _create_file_request=_create_file_request, ) else: - apply_team_provider_credentials( + await apply_team_provider_credentials( data=cast(dict, _create_file_request), # cast-ok: TypedDict is a plain dict at runtime; merged in place llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -1131,7 +1131,7 @@ async def get_file_content( ) if not should_route: - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -1397,7 +1397,7 @@ async def get_file( # Remove file_id from data to avoid "multiple values for keyword argument" error # data was initialized with {"file_id": file_id} data.pop("file_id", None) - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -1616,7 +1616,7 @@ async def delete_file( ) else: data.pop("file_id", None) - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, @@ -2018,7 +2018,7 @@ async def list_files( ) else: resolved_custom_llm_provider: Final = custom_llm_provider or "openai" - apply_team_provider_credentials( + await apply_team_provider_credentials( data=data, llm_router=llm_router, user_api_key_dict=user_api_key_dict, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 34fd1efefbb..ada3035baf8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -440,6 +440,7 @@ from litellm.proxy.auth.model_checks import ( get_key_models, get_mcp_server_ids, get_team_models, + run_model_discovery, ) from litellm.proxy.auth.password_policy import validate_password_not_breached, validate_password_policy from litellm.proxy.auth.user_api_key_auth import ( # noqa: F401, RUF100 # legacy module exports @@ -12418,7 +12419,7 @@ async def model_list( # Get complete model list including wildcard routes if requested from litellm.proxy.auth.model_checks import get_complete_model_list - all_models = await asyncio.to_thread( + all_models = await run_model_discovery( get_complete_model_list, key_models=[], team_models=[], @@ -17854,7 +17855,7 @@ async def model_info_v1( alias_models: Final = copy.deepcopy(llm_router.get_model_list_from_model_alias()) all_models.extend(alias_models) - all_models = await asyncio.to_thread(expand_wildcard_deployments_for_model_info, all_models, llm_router) + all_models = await run_model_discovery(expand_wildcard_deployments_for_model_info, all_models, llm_router) allowed_model_names: Final = await _get_v1_model_info_allowed_model_names_with_access_groups( user_api_key_dict=user_api_key_dict, @@ -18189,7 +18190,7 @@ async def model_group_info( LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, ) all_models_str: Final = ( - await asyncio.to_thread( + await run_model_discovery( get_complete_model_list, key_models=(), team_models=(), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d6916d5be04..a2f0b202dfc 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -9499,7 +9499,7 @@ async def _agent_access_group_visible_models( ) -> frozenset[str] | None: """Models an agent key may still list once its attached access groups cap it, ``None`` when nothing caps it, so ``/v1/models`` never advertises a model the same key would be denied on.""" - from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models + from litellm.proxy.auth.model_checks import get_complete_model_list, get_team_models, run_model_discovery if not user_api_key_dict.agent_id: return None @@ -9519,7 +9519,7 @@ async def _agent_access_group_visible_models( if not granted: return frozenset() return frozenset( - await asyncio.to_thread( + await run_model_discovery( get_complete_model_list, key_models=granted, team_models=(), @@ -9575,6 +9575,7 @@ async def get_available_models_for_user( get_complete_model_list, get_key_models, get_team_models, + run_model_discovery, ) # Get proxy model list and access groups @@ -9655,7 +9656,7 @@ async def get_available_models_for_user( ) # Get complete model list - complete_models: Final = await asyncio.to_thread( + complete_models: Final = await run_model_discovery( get_complete_model_list, key_models=granted_key_models, team_models=granted_team_models, diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 4033d2a30f5..058ef08aef4 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -588,6 +588,59 @@ async def test_a_hung_wildcard_listing_leaves_the_event_loop_free_for_other_requ assert "hosted_vllm/qwen2.5" in await listing +@respx.mock +def test_a_hung_wildcard_listing_leaves_the_default_executor_free_for_completion_calls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import asyncio + from concurrent.futures import ThreadPoolExecutor + + import litellm + from litellm import Router + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import get_available_models_for_user + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + listing_started: Final = threading.Event() + completion_served: Final = threading.Event() + + def listing_that_answers_once_a_completion_is_served(request: httpx.Request) -> httpx.Response: + listing_started.set() + if completion_served.wait(timeout=3): + return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + return httpx.Response(503) + + respx.get("http://vllm-default-executor.test:8000/v1/models").mock( + side_effect=listing_that_answers_once_a_completion_is_served + ) + router: Final = Router( + model_list=[ + { + "model_name": "hosted_vllm/*", + "litellm_params": {"model": "hosted_vllm/*", "api_base": "http://vllm-default-executor.test:8000/v1"}, + } + ] + ) + + async def list_models_while_a_completion_runs_on_the_default_executor() -> list[str]: + loop: Final = asyncio.get_running_loop() + loop.set_default_executor(ThreadPoolExecutor(max_workers=1)) + listing: Final = asyncio.create_task( + get_available_models_for_user( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=[]), + llm_router=router, + general_settings={}, + user_model=None, + ) + ) + with ThreadPoolExecutor(max_workers=1) as waiter: + assert await loop.run_in_executor(waiter, listing_started.wait, 3) + await asyncio.wait_for(loop.run_in_executor(None, completion_served.set), timeout=1) + return await listing + + assert "hosted_vllm/qwen2.5" in asyncio.run(list_models_while_a_completion_runs_on_the_default_executor()) + + @respx.mock def test_hosted_vllm_wildcard_listing_sends_the_deployment_extra_headers(monkeypatch: pytest.MonkeyPatch) -> None: import litellm diff --git a/tests/unit/proxy/hooks/test_batch_file_validation.py b/tests/unit/proxy/hooks/test_batch_file_validation.py index 8eeee2e6468..dca7b0c7071 100644 --- a/tests/unit/proxy/hooks/test_batch_file_validation.py +++ b/tests/unit/proxy/hooks/test_batch_file_validation.py @@ -1274,7 +1274,8 @@ def test_should_not_skip_and_reuses_descriptors_when_limits_present(): assert returned is descriptors -def test_resolve_fetch_params_uses_request_model_credentials(): +@pytest.mark.asyncio +async def test_resolve_fetch_params_uses_request_model_credentials(): rate_limiter = _make_rate_limiter() with ( patch("litellm.proxy.proxy_server.llm_router", MagicMock()), @@ -1288,7 +1289,7 @@ def test_resolve_fetch_params_uses_request_model_credentials(): ), ): provider_file_id, fetch_kwargs = ( - rate_limiter._resolve_batch_input_file_fetch_params( + await rate_limiter._resolve_batch_input_file_fetch_params( file_id="file-plain-openai", custom_llm_provider="openai", data={"model": "my-vllm-batch"}, @@ -1300,7 +1301,8 @@ def test_resolve_fetch_params_uses_request_model_credentials(): assert fetch_kwargs["api_base"] == "http://vllm:8000/v1" -def test_resolve_fetch_params_fails_open_on_credential_lookup_error(): +@pytest.mark.asyncio +async def test_resolve_fetch_params_fails_open_on_credential_lookup_error(): rate_limiter = _make_rate_limiter() with ( patch("litellm.proxy.proxy_server.llm_router", MagicMock()), @@ -1310,7 +1312,7 @@ def test_resolve_fetch_params_fails_open_on_credential_lookup_error(): ), ): provider_file_id, fetch_kwargs = ( - rate_limiter._resolve_batch_input_file_fetch_params( + await rate_limiter._resolve_batch_input_file_fetch_params( file_id="file-plain-openai", custom_llm_provider="openai", data={"model": "my-vllm-batch"}, @@ -1320,7 +1322,8 @@ def test_resolve_fetch_params_fails_open_on_credential_lookup_error(): assert fetch_kwargs == {"custom_llm_provider": "openai"} -def test_resolve_fetch_params_model_embedded_fails_open_on_credential_error(): +@pytest.mark.asyncio +async def test_resolve_fetch_params_model_embedded_fails_open_on_credential_error(): import base64 rate_limiter = _make_rate_limiter() @@ -1342,7 +1345,7 @@ def test_resolve_fetch_params_model_embedded_fails_open_on_credential_error(): ), ): provider_file_id, fetch_kwargs = ( - rate_limiter._resolve_batch_input_file_fetch_params( + await rate_limiter._resolve_batch_input_file_fetch_params( file_id=encoded_file_id, custom_llm_provider="openai", data={}, diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index f6c6b51fc93..dc86253e49a 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -1,9 +1,13 @@ +import asyncio +import threading from datetime import datetime, timezone from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, MagicMock +import httpx import pytest +import respx from fastapi import HTTPException from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles @@ -764,26 +768,77 @@ def test_a_request_key_that_cannot_be_an_http_header_is_a_bad_request(api_key: s assert api_key not in str(refused.value.detail) +@pytest.mark.asyncio +@respx.mock +async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_event_loop_free( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + listing_started: Final = threading.Event() + other_request_served: Final = threading.Event() + listing_answered: Final = threading.Event() + + def listing_that_answers_once_another_request_is_served(request: httpx.Request) -> httpx.Response: + listing_started.set() + if other_request_served.wait(timeout=3): + listing_answered.set() + return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + return httpx.Response(503) + + respx.get("http://vllm-credential-lookup.test:8000/v1/models").mock( + side_effect=listing_that_answers_once_another_request_is_served + ) + data: Final[dict[str, object]] = {} + lookup: Final = asyncio.create_task( + apply_team_provider_credentials( + data=data, + llm_router=Router( + model_list=[ + { + "model_name": "hosted_vllm/*", + "litellm_params": { + "model": "hosted_vllm/*", + "api_base": "http://vllm-credential-lookup.test:8000/v1", + }, + } + ] + ), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + custom_llm_provider="hosted_vllm", + ) + ) + assert await asyncio.to_thread(listing_started.wait, 3) + other_request_served.set() + await lookup + + assert listing_answered.is_set() + assert data.get("api_base") == "http://vllm-credential-lookup.test:8000/v1" + + def _apply_team_a_openai_credentials(data: dict[str, object]) -> None: - apply_team_provider_credentials( - data=data, - llm_router=Router( - model_list=[ - { - "model_name": "openai/*", - "litellm_params": { - "model": "openai/*", - "api_key": "team-openai-key", - "api_base": "https://team.example/v1", - }, - "model_info": {"id": "team-a-openai", "team_id": "team-a"}, - } - ] - ), - user_api_key_dict=UserAPIKeyAuth( - api_key="test-key", user_id="test-user", team_id="team-a", team_models=["openai/*"], models=["gpt-4o"] - ), - custom_llm_provider="openai", + asyncio.run( + apply_team_provider_credentials( + data=data, + llm_router=Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "team-openai-key", + "api_base": "https://team.example/v1", + }, + "model_info": {"id": "team-a-openai", "team_id": "team-a"}, + } + ] + ), + user_api_key_dict=UserAPIKeyAuth( + api_key="test-key", user_id="test-user", team_id="team-a", team_models=["openai/*"], models=["gpt-4o"] + ), + custom_llm_provider="openai", + ) ) From 1f03ebc98308f8623db7929e64877a6da5cd416a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:01:10 -0700 Subject: [PATCH 12/14] fix(proxy): stop the file credential lookup at the first match before listing wildcard models The fallback lookup now walks the allowed model names lazily, in the same order as before, so a hung vLLM server is listed only when no configured name serves the requested provider. The vLLM auth headers no longer mutate the caller's headers dict. --- litellm/llms/vllm/common_utils.py | 3 +- litellm/proxy/auth/model_checks.py | 159 ++++++++++-------- .../openai_files_endpoints/common_utils.py | 40 ++--- .../test_files_common_utils.py | 43 ++++- 4 files changed, 151 insertions(+), 94 deletions(-) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index 2340c6ba133..62efa89e593 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -56,8 +56,7 @@ class VLLMModelInfo(BaseLLMModelInfo): api_key: str | None = None, api_base: str | None = None, ) -> dict: - headers.update(_auth_headers(api_key)) - return headers + return {**headers, **_auth_headers(api_key)} @staticmethod def get_api_base(api_base: str | None = None) -> str: diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index e6f6d60855a..a997b53d265 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -6,7 +6,7 @@ import copy import functools import itertools import re -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from typing import Final, ParamSpec, TypeVar @@ -240,46 +240,85 @@ def get_complete_model_list( If list contains wildcard -> return known provider models """ - unique_models: Final = [] - - def append_unique(models): - for model in models: - if model not in unique_models and model != SpecialModelNames.no_default_models.value: - unique_models.append(model) - - if key_models: - append_unique(key_models) - elif team_models: - append_unique(team_models) - else: - append_unique(proxy_model_list) - if include_model_access_groups: - append_unique(list(model_access_groups.keys())) # TODO: keys order - - if user_model: - append_unique([user_model]) - - if infer_model_from_keys: - valid_models: Final = get_valid_models() - append_unique(valid_models) - if only_model_access_groups: - model_access_groups_to_return: Final[list[str]] = [] - for model in unique_models: - if model in model_access_groups: - model_access_groups_to_return.append(model) - return model_access_groups_to_return - - all_wildcard_models: Final = _get_wildcard_models( - unique_models=unique_models, - return_wildcard_routes=return_wildcard_routes, - llm_router=llm_router, - team_id=team_id, + return [ + model + for model in _unexpanded_model_list( + key_models=key_models, + team_models=team_models, + proxy_model_list=proxy_model_list, + user_model=user_model, + infer_model_from_keys=infer_model_from_keys, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + ) + if model in model_access_groups + ] + return list( + iter_complete_model_list( + key_models=key_models, + team_models=team_models, + proxy_model_list=proxy_model_list, + user_model=user_model, + infer_model_from_keys=infer_model_from_keys, + return_wildcard_routes=return_wildcard_routes, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + team_id=team_id, + ) ) - complete_model_list: Final = unique_models + all_wildcard_models - return complete_model_list +def _unexpanded_model_list( + key_models: Sequence[str], + team_models: Sequence[str], + proxy_model_list: Sequence[str], + user_model: str | None, + infer_model_from_keys: bool | None, + model_access_groups: Mapping[str, Sequence[str]], + include_model_access_groups: bool | None, +) -> list[str]: + granted_model_names: Final = ( + key_models + or team_models + or ( + *proxy_model_list, + *(model_access_groups if include_model_access_groups else ()), + *((user_model,) if user_model else ()), + *(get_valid_models() if infer_model_from_keys else ()), + ) + ) + return [model for model in dict.fromkeys(granted_model_names) if model != SpecialModelNames.no_default_models.value] + + +def iter_complete_model_list( + key_models: Sequence[str], + team_models: Sequence[str], + proxy_model_list: Sequence[str], + user_model: str | None, + infer_model_from_keys: bool | None, + return_wildcard_routes: bool | None, + llm_router: Router | None, + model_access_groups: Mapping[str, Sequence[str]], + include_model_access_groups: bool | None, + team_id: str | None, +) -> Iterator[str]: + unexpanded_models: Final = _unexpanded_model_list( + key_models=key_models, + team_models=team_models, + proxy_model_list=proxy_model_list, + user_model=user_model, + infer_model_from_keys=infer_model_from_keys, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + ) + yield from (model for model in unexpanded_models if not _check_wildcard_routing(model=model)) + routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) + for wildcard_route in (model for model in unexpanded_models if _check_wildcard_routing(model=model)): + if return_wildcard_routes: + yield wildcard_route + yield from _wildcard_route_models(wildcard_route, llm_router, team_id, routes_to_model) def _hydrate_litellm_credential_name( @@ -558,36 +597,22 @@ def expand_wildcard_deployments_for_model_info( return expanded -def _get_wildcard_models( - unique_models: list[str], - return_wildcard_routes: bool | None = False, - llm_router: Router | None = None, - team_id: str | None = None, -) -> list[str]: - wildcard_routes: Final = tuple(model for model in unique_models if _check_wildcard_routing(model=model)) - routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) - all_wildcard_models: Final = [] - for model in wildcard_routes: - if return_wildcard_routes: - all_wildcard_models.append(model) - - model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None - if model_list: - for router_model in model_list: - all_wildcard_models.extend( - get_known_models_from_wildcard( - wildcard_model=model, - litellm_params=LiteLLM_Params(**router_model["litellm_params"]), - routes_to_model=routes_to_model, - ) - ) - else: - all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None)) - - for model in frozenset(wildcard_routes): - unique_models.remove(model) - - return all_wildcard_models +def _wildcard_route_models( + wildcard_route: str, + llm_router: Router | None, + team_id: str | None, + routes_to_model: Callable[[str, str], bool], +) -> Iterator[str]: + router_models: Final = llm_router.get_model_list(model_name=wildcard_route, team_id=team_id) if llm_router else None + if not router_models: + yield from get_known_models_from_wildcard(wildcard_model=wildcard_route, litellm_params=None) + return + for router_model in router_models: + yield from get_known_models_from_wildcard( + wildcard_model=wildcard_route, + litellm_params=LiteLLM_Params(**router_model["litellm_params"]), + routes_to_model=routes_to_model, + ) def get_all_fallbacks( diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 907fd11347c..0073528c61b 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -559,7 +559,7 @@ def get_team_provider_credentials( return None from litellm.proxy._types import SpecialModelNames - from litellm.proxy.auth.model_checks import get_complete_model_list, get_key_models + from litellm.proxy.auth.model_checks import get_key_models, iter_complete_model_list team_id: Final = user_api_key_dict.team_id team_models: Final = user_api_key_dict.team_models or [] @@ -637,35 +637,29 @@ def get_team_provider_credentials( # effective allowlist (sentinels and access groups already expanded by # get_key_models) wins when set; otherwise the team's allowlist applies. # The all-proxy-models sentinel isn't expanded by - # get_complete_model_list, so normalize it to an empty allowlist, which + # iter_complete_model_list, so normalize it to an empty allowlist, which # defers to the team-scoped proxy model list. A team or key with a # restricted allowlist (e.g. anthropic only) therefore never resolves # another provider's key. grants_all_models: Final = SpecialModelNames.all_proxy_models.value in team_models effective_team_models: Final = [] if grants_all_models else team_models - models_to_try: Final = list( - dict.fromkeys( - get_complete_model_list( - key_models=list(key_model_allowlist), - team_models=effective_team_models, - proxy_model_list=proxy_model_list, - user_model=None, - infer_model_from_keys=False, - return_wildcard_routes=True, - llm_router=llm_router, - model_access_groups=model_access_groups, - include_model_access_groups=True, - team_id=team_id, - ) - ) + models_to_try: Final = iter_complete_model_list( + key_models=key_model_allowlist, + team_models=effective_team_models, + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + return_wildcard_routes=True, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=True, + team_id=team_id, + ) + return next( + (credentials for credentials in map(_provider_credentials, models_to_try) if credentials is not None), + None, ) - for model_name in models_to_try: - credentials = _provider_credentials(model_name) - if credentials is not None: - return credentials - - return None HTTP_HEADER_VALUE: Final = re.compile(r"[\x21-\x7e]+(?:[ \t]+[\x21-\x7e]+)*") diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index dc86253e49a..5f2f4c465e7 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -806,7 +806,7 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve ] ), user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), - custom_llm_provider="hosted_vllm", + custom_llm_provider="openai", ) ) assert await asyncio.to_thread(listing_started.wait, 3) @@ -814,7 +814,46 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve await lookup assert listing_answered.is_set() - assert data.get("api_base") == "http://vllm-credential-lookup.test:8000/v1" + assert data == {} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "custom_llm_provider, vllm_api_base, applied_credential", + [ + ("openai", "http://vllm-beside-openai.test:8000/v1", ("api_key", "openai-key")), + ("hosted_vllm", "http://vllm-route-match.test:8000/v1", ("api_base", "http://vllm-route-match.test:8000/v1")), + ], +) +async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wildcard_server( + monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, vllm_api_base: str, applied_credential: tuple[str, str] +) -> None: + import litellm + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + data: Final[dict[str, object]] = {} + + with respx.mock(assert_all_called=False) as vllm_server: + listing: Final = vllm_server.get(f"{vllm_api_base}/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + await apply_team_provider_credentials( + data=data, + llm_router=Router( + model_list=[ + { + "model_name": "hosted_vllm/*", + "litellm_params": {"model": "hosted_vllm/*", "api_base": vllm_api_base}, + }, + {"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "openai-key"}}, + ] + ), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + custom_llm_provider=custom_llm_provider, + ) + + assert not listing.called + assert data.get(applied_credential[0]) == applied_credential[1] def _apply_team_a_openai_credentials(data: dict[str, object]) -> None: From 83d52c764652fccc6c252586b955dcb102c840a4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:33:57 -0700 Subject: [PATCH 13/14] fix(proxy): try every configured wildcard route before listing any wildcard's models in the file credential lookup --- litellm/proxy/auth/model_checks.py | 29 +++++++++++++++++-- .../openai_files_endpoints/common_utils.py | 10 ++----- .../test_files_common_utils.py | 28 ++++++++++++++---- 3 files changed, 53 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index a997b53d265..7739cd9941e 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -255,7 +255,7 @@ def get_complete_model_list( if model in model_access_groups ] return list( - iter_complete_model_list( + _iter_complete_model_list( key_models=key_models, team_models=team_models, proxy_model_list=proxy_model_list, @@ -292,7 +292,7 @@ def _unexpanded_model_list( return [model for model in dict.fromkeys(granted_model_names) if model != SpecialModelNames.no_default_models.value] -def iter_complete_model_list( +def _iter_complete_model_list( key_models: Sequence[str], team_models: Sequence[str], proxy_model_list: Sequence[str], @@ -321,6 +321,31 @@ def iter_complete_model_list( yield from _wildcard_route_models(wildcard_route, llm_router, team_id, routes_to_model) +def iter_model_names_expanding_wildcards_last( + key_models: Sequence[str], + team_models: Sequence[str], + proxy_model_list: Sequence[str], + llm_router: Router | None, + model_access_groups: Mapping[str, Sequence[str]], + team_id: str | None, +) -> Iterator[str]: + unexpanded_models: Final = _unexpanded_model_list( + key_models=key_models, + team_models=team_models, + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + model_access_groups=model_access_groups, + include_model_access_groups=True, + ) + wildcard_routes: Final = tuple(model for model in unexpanded_models if _check_wildcard_routing(model=model)) + yield from (model for model in unexpanded_models if not _check_wildcard_routing(model=model)) + yield from wildcard_routes + routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) + for wildcard_route in wildcard_routes: + yield from _wildcard_route_models(wildcard_route, llm_router, team_id, routes_to_model) + + def _hydrate_litellm_credential_name( litellm_params: LiteLLM_Params | None, ) -> LiteLLM_Params | None: diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 0073528c61b..371456f3991 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -559,7 +559,7 @@ def get_team_provider_credentials( return None from litellm.proxy._types import SpecialModelNames - from litellm.proxy.auth.model_checks import get_key_models, iter_complete_model_list + from litellm.proxy.auth.model_checks import get_key_models, iter_model_names_expanding_wildcards_last team_id: Final = user_api_key_dict.team_id team_models: Final = user_api_key_dict.team_models or [] @@ -637,23 +637,19 @@ def get_team_provider_credentials( # effective allowlist (sentinels and access groups already expanded by # get_key_models) wins when set; otherwise the team's allowlist applies. # The all-proxy-models sentinel isn't expanded by - # iter_complete_model_list, so normalize it to an empty allowlist, which + # iter_model_names_expanding_wildcards_last, so normalize it to an empty allowlist, which # defers to the team-scoped proxy model list. A team or key with a # restricted allowlist (e.g. anthropic only) therefore never resolves # another provider's key. grants_all_models: Final = SpecialModelNames.all_proxy_models.value in team_models effective_team_models: Final = [] if grants_all_models else team_models - models_to_try: Final = iter_complete_model_list( + models_to_try: Final = iter_model_names_expanding_wildcards_last( key_models=key_model_allowlist, team_models=effective_team_models, proxy_model_list=proxy_model_list, - user_model=None, - infer_model_from_keys=False, - return_wildcard_routes=True, llm_router=llm_router, model_access_groups=model_access_groups, - include_model_access_groups=True, team_id=team_id, ) return next( diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index 5f2f4c465e7..6d1fb00308f 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -819,14 +819,29 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve @pytest.mark.asyncio @pytest.mark.parametrize( - "custom_llm_provider, vllm_api_base, applied_credential", + "custom_llm_provider, vllm_api_base, openai_deployment, applied_credential", [ - ("openai", "http://vllm-beside-openai.test:8000/v1", ("api_key", "openai-key")), - ("hosted_vllm", "http://vllm-route-match.test:8000/v1", ("api_base", "http://vllm-route-match.test:8000/v1")), + ("openai", "http://vllm-beside-openai.test:8000/v1", ("gpt-4o", "openai/gpt-4o"), ("api_key", "openai-key")), + ( + "openai", + "http://vllm-before-openai-wildcard.test:8000/v1", + ("openai/*", "openai/*"), + ("api_key", "openai-key"), + ), + ( + "hosted_vllm", + "http://vllm-route-match.test:8000/v1", + ("gpt-4o", "openai/gpt-4o"), + ("api_base", "http://vllm-route-match.test:8000/v1"), + ), ], ) async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wildcard_server( - monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, vllm_api_base: str, applied_credential: tuple[str, str] + monkeypatch: pytest.MonkeyPatch, + custom_llm_provider: str, + vllm_api_base: str, + openai_deployment: tuple[str, str], + applied_credential: tuple[str, str], ) -> None: import litellm @@ -845,7 +860,10 @@ async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wi "model_name": "hosted_vllm/*", "litellm_params": {"model": "hosted_vllm/*", "api_base": vllm_api_base}, }, - {"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "openai-key"}}, + { + "model_name": openai_deployment[0], + "litellm_params": {"model": openai_deployment[1], "api_key": "openai-key"}, + }, ] ), user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), From 42ee8014092e8e53eeeaf352f6c2fd51aff9b026 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:30:22 -0700 Subject: [PATCH 14/14] fix(proxy): answer model lists from cached listings first and share one listing per server --- litellm/constants.py | 1 + litellm/proxy/auth/model_checks.py | 7 ++- .../openai_files_endpoints/common_utils.py | 17 ++++++ litellm/router.py | 20 +++--- litellm/utils.py | 57 ++++++++++++++--- tests/unit/proxy/auth/test_model_checks.py | 61 +++++++++++++++++++ .../test_files_common_utils.py | 15 +++-- tests/unit/test_utils.py | 37 +++++++++++ 8 files changed, 194 insertions(+), 21 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index d2ac3d62377..d416be0586f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -613,6 +613,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int( ) # timeout for adjacent services (e.g. jwt auth) PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS: Final = 60 PROVIDER_MODEL_DISCOVERY_MAX_THREADS: Final = 4 +PROVIDER_MODEL_DISCOVERY_LISTING_LOCK_STRIPES: Final = 64 MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000) MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048) MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024 diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index 7739cd9941e..163ec2e3141 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -22,7 +22,7 @@ from litellm.router_utils.fallback_event_handlers import get_fallback_model_grou from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager, get_valid_models +from litellm.utils import ProviderConfigManager, answer_from_cached_provider_listings, get_valid_models LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders) @@ -40,6 +40,11 @@ async def run_model_discovery( *args: _P.args, **kwargs: _P.kwargs, # kwargs-ok: forwards the discovery function's own arguments ) -> _R: + cached_answer, needs_provider_listing = answer_from_cached_provider_listings( + functools.partial(discover, *args, **kwargs) + ) + if not needs_provider_listing: + return cached_answer context: Final = contextvars.copy_context() return await asyncio.get_running_loop().run_in_executor( _MODEL_DISCOVERY_EXECUTOR, lambda: context.run(discover, *args, **kwargs) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 371456f3991..40837919025 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -509,6 +509,18 @@ _UNKNOWN_CREATION_TIME: Final = datetime.max.replace(tzinfo=timezone.utc) class _RouterDeployment(TypedDict): model_name: ReadOnly[NotRequired[str]] model_info: ReadOnly[Mapping[str, object]] + litellm_params: ReadOnly[Mapping[str, object]] + + +def _deployment_serves_provider(llm_router: "Router", deployment: _RouterDeployment, custom_llm_provider: str) -> bool: + litellm_params: Final = deployment["litellm_params"] + model: Final = litellm_params.get("model") + deployment_provider: Final = litellm_params.get("custom_llm_provider") + return ( + isinstance(model, str) + and llm_router.credential_provider(deployment_provider if isinstance(deployment_provider, str) else None, model) + == custom_llm_provider + ) def _team_deployment_rank(model_info: Mapping[str, object], deployment_id: str) -> tuple[datetime, str]: @@ -632,6 +644,11 @@ def get_team_provider_credentials( if not include_accessible_deployments: return None + if not any( + _deployment_serves_provider(llm_router, deployment, custom_llm_provider) + for deployment in llm_router.model_list or [] + ): + return None # 2. Fall back to deployments the caller is allowed to access. The key's # effective allowlist (sentinels and access groups already expanded by diff --git a/litellm/router.py b/litellm/router.py index 02ad6275b82..992996464ff 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -12379,18 +12379,20 @@ class Router: credentials.pop("litellm_credential_name", None) credentials["model"] = deployment.litellm_params.model - - # Add custom_llm_provider - if deployment.litellm_params.custom_llm_provider: - credentials["custom_llm_provider"] = deployment.litellm_params.custom_llm_provider - elif "/" in deployment.litellm_params.model: - # Extract provider from "provider/model" format - credentials["custom_llm_provider"] = deployment.litellm_params.model.split("/")[0] - else: - credentials["custom_llm_provider"] = "openai" # default + credentials["custom_llm_provider"] = self.credential_provider( + deployment.litellm_params.custom_llm_provider, deployment.litellm_params.model + ) return credentials + @staticmethod + def credential_provider(custom_llm_provider: str | None, model: str) -> str: + if custom_llm_provider: + return custom_llm_provider + if "/" in model: + return model.split("/")[0] + return "openai" + @overload def get_router_model_info( self, diff --git a/litellm/utils.py b/litellm/utils.py index 2350e88d11d..bf0c5ca958a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -79,6 +79,7 @@ from litellm.constants import ( MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE, NON_INFERENCE_CALL_TYPES, OPENAI_EMBEDDING_PARAMS, + PROVIDER_MODEL_DISCOVERY_LISTING_LOCK_STRIPES, PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS, PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, @@ -8371,6 +8372,13 @@ class AvailableModelsCache(InMemoryCache): super().__init__(ttl_seconds, max_size, clock=clock) self._env_hash: str | None = None self._listing_lock = threading.Lock() + self._single_flight_locks: Final = tuple( + threading.RLock() for _ in range(PROVIDER_MODEL_DISCOVERY_LISTING_LOCK_STRIPES) + ) + + def single_flight_lock(self, custom_llm_provider: str, litellm_params: LiteLLM_Params | None) -> threading.RLock: + cache_key: Final = self._get_cache_key(custom_llm_provider, litellm_params) + return self._single_flight_locks[hash(cache_key) % len(self._single_flight_locks)] def _get_env_hash(self) -> str: """Create a hash of relevant environment variables""" @@ -8451,6 +8459,22 @@ class AvailableModelsCache(InMemoryCache): # Global cache instance _model_cache: Final = AvailableModelsCache() +_UNCACHED_PROVIDER_LISTING: Final[contextvars.ContextVar[threading.Event | None]] = contextvars.ContextVar( + "uncached_provider_listing", default=None +) +_Answer: Final = TypeVar("_Answer") + + +def answer_from_cached_provider_listings(compute: Callable[[], _Answer]) -> tuple[_Answer, bool]: + uncached_listing: Final = threading.Event() + answer: Final = contextvars.copy_context().run(_answer_marking_uncached_listings, uncached_listing, compute) + return answer, uncached_listing.is_set() + + +def _answer_marking_uncached_listings(uncached_listing: threading.Event, compute: Callable[[], _Answer]) -> _Answer: + _UNCACHED_PROVIDER_LISTING.set(uncached_listing) + return compute() + def _infer_valid_provider_from_env_vars( custom_llm_provider: str | None = None, @@ -8504,21 +8528,40 @@ def _get_valid_models_from_provider_api( if cached_result is not None: return cached_result + uncached_listing: Final = _UNCACHED_PROVIDER_LISTING.get() + if uncached_listing is not None: + uncached_listing.set() + return [] + with model_cache.single_flight_lock(custom_llm_provider, litellm_params): + return _list_provider_models_unless_cached( + provider_config, custom_llm_provider, litellm_params, model_cache + ) + except Exception as e: + verbose_logger.warning("Error getting valid models: %s", e) + return [] + + +def _list_provider_models_unless_cached( + provider_config: BaseLLMModelInfo, + custom_llm_provider: str, + litellm_params: LiteLLM_Params | None, + model_cache: AvailableModelsCache, +) -> list[str]: + cached_while_waiting: Final = model_cache.get_cached_model_info(custom_llm_provider, litellm_params) + if cached_while_waiting is not None: + return cached_while_waiting + try: models: Final = provider_config.discover_models( litellm_params=_provider_listing_params(custom_llm_provider, litellm_params) ) - - model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) - return models except httpx.TimeoutException as e: - verbose_logger.warning("Timed out getting valid models: %s", e) + verbose_logger.warning("Error getting valid models: %s", e) model_cache.cache_timed_out_listing_unless_listed( custom_llm_provider, litellm_params, ttl=PROVIDER_MODEL_DISCOVERY_TIMEOUT_CACHE_TTL_SECONDS ) return model_cache.get_cached_model_info(custom_llm_provider, litellm_params) or [] - except Exception as e: - verbose_logger.warning("Error getting valid models: %s", e) - return [] + model_cache.set_cached_model_info(custom_llm_provider, litellm_params, models) + return models def get_valid_models( diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 058ef08aef4..6ed87afc77e 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -588,6 +588,67 @@ async def test_a_hung_wildcard_listing_leaves_the_event_loop_free_for_other_requ assert "hosted_vllm/qwen2.5" in await listing +@pytest.mark.asyncio +@respx.mock +async def test_a_model_list_that_needs_no_listing_answers_while_hung_listings_hold_every_discovery_thread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import asyncio + + import litellm + from litellm import Router + from litellm.constants import PROVIDER_MODEL_DISCOVERY_MAX_THREADS + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.utils import get_available_models_for_user + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + listing_started: Final = threading.Event() + hung_server_released: Final = threading.Event() + + def hung_listing(request: httpx.Request) -> httpx.Response: + listing_started.set() + hung_server_released.wait(timeout=10) + return httpx.Response(503) + + respx.get("http://vllm-holding-every-thread.test:8000/v1/models").mock(side_effect=hung_listing) + router: Final = Router( + model_list=[ + { + "model_name": "stuck/*", + "litellm_params": { + "model": "hosted_vllm/*", + "api_base": "http://vllm-holding-every-thread.test:8000/v1", + }, + }, + { + "model_name": "qwen-open", + "litellm_params": {"model": "hosted_vllm/qwen2.5", "api_base": "http://vllm-open.test:8000/v1"}, + }, + ] + ) + + def list_models(models: list[str]) -> asyncio.Task[list[str]]: + return asyncio.create_task( + get_available_models_for_user( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=models), + llm_router=router, + general_settings={}, + user_model=None, + ) + ) + + try: + admin_listings: Final = tuple(list_models([]) for _ in range(PROVIDER_MODEL_DISCOVERY_MAX_THREADS)) + assert await asyncio.to_thread(listing_started.wait, 3) + await asyncio.sleep(0.1) + + assert await asyncio.wait_for(list_models(["qwen-open"]), timeout=2) == ["qwen-open"] + finally: + hung_server_released.set() + for admin_models in await asyncio.gather(*admin_listings): + assert "qwen-open" in admin_models + + @respx.mock def test_a_hung_wildcard_listing_leaves_the_default_executor_free_for_completion_calls( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index 6d1fb00308f..15ce9841cde 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -802,10 +802,11 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve "model": "hosted_vllm/*", "api_base": "http://vllm-credential-lookup.test:8000/v1", }, - } + }, + {"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "openai-key"}}, ] ), - user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", models=["hosted_vllm/*"]), custom_llm_provider="openai", ) ) @@ -834,14 +835,20 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve ("gpt-4o", "openai/gpt-4o"), ("api_base", "http://vllm-route-match.test:8000/v1"), ), + ( + "anthropic", + "http://vllm-beside-no-anthropic-deployment.test:8000/v1", + ("gpt-4o", "openai/gpt-4o"), + ("api_key", None), + ), ], ) -async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wildcard_server( +async def test_a_credential_lookup_never_lists_a_wildcard_server_whose_models_cannot_change_the_answer( monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, vllm_api_base: str, openai_deployment: tuple[str, str], - applied_credential: tuple[str, str], + applied_credential: tuple[str, str | None], ) -> None: import litellm diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 9e41b61ca5c..75cdf4babce 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10616,6 +10616,43 @@ def test_a_timed_out_provider_model_listing_is_retried_only_after_its_timeout_tt assert models_route.call_count == 2 +@respx.mock +def test_concurrent_listings_of_one_server_share_a_single_request() -> None: + import time + + from litellm.llms.vllm.common_utils import HostedVLLMModelInfo + from litellm.types.router import LiteLLM_Params + from litellm.utils import AvailableModelsCache, _get_valid_models_from_provider_api + + listing_started: Final = threading.Event() + server_released: Final = threading.Event() + + def slow_listing(request: httpx.Request) -> httpx.Response: + listing_started.set() + server_released.wait(timeout=5) + return httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + + models_route: Final = respx.get("http://vllm-slow.test:8000/v1/models").mock(side_effect=slow_listing) + litellm_params: Final = LiteLLM_Params(model="hosted_vllm/*", api_base="http://vllm-slow.test:8000/v1") + model_cache: Final = AvailableModelsCache(clock=Mock(return_value=1000.0)) + + def list_models() -> list[str]: + return _get_valid_models_from_provider_api( + HostedVLLMModelInfo(), "hosted_vllm", litellm_params, model_cache=model_cache + ) + + with ThreadPoolExecutor(max_workers=4) as callers: + listings: Final = tuple(callers.submit(list_models) for _ in range(4)) + try: + assert listing_started.wait(timeout=3) + time.sleep(0.3) + finally: + server_released.set() + assert [listing.result(timeout=5) for listing in listings] == [["qwen2.5"]] * 4 + + assert models_route.call_count == 1 + + @respx.mock def test_a_listing_that_times_out_keeps_the_models_a_concurrent_listing_cached() -> None: from litellm.llms.vllm.common_utils import HostedVLLMModelInfo