From 1f03ebc98308f8623db7929e64877a6da5cd416a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 07:01:10 -0700 Subject: [PATCH] fix(proxy): stop the file credential lookup at the first match before listing wildcard models The fallback lookup now walks the allowed model names lazily, in the same order as before, so a hung vLLM server is listed only when no configured name serves the requested provider. The vLLM auth headers no longer mutate the caller's headers dict. --- litellm/llms/vllm/common_utils.py | 3 +- litellm/proxy/auth/model_checks.py | 159 ++++++++++-------- .../openai_files_endpoints/common_utils.py | 40 ++--- .../test_files_common_utils.py | 43 ++++- 4 files changed, 151 insertions(+), 94 deletions(-) diff --git a/litellm/llms/vllm/common_utils.py b/litellm/llms/vllm/common_utils.py index 2340c6ba133..62efa89e593 100644 --- a/litellm/llms/vllm/common_utils.py +++ b/litellm/llms/vllm/common_utils.py @@ -56,8 +56,7 @@ class VLLMModelInfo(BaseLLMModelInfo): api_key: str | None = None, api_base: str | None = None, ) -> dict: - headers.update(_auth_headers(api_key)) - return headers + return {**headers, **_auth_headers(api_key)} @staticmethod def get_api_base(api_base: str | None = None) -> str: diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index e6f6d60855a..a997b53d265 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -6,7 +6,7 @@ import copy import functools import itertools import re -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from typing import Final, ParamSpec, TypeVar @@ -240,46 +240,85 @@ def get_complete_model_list( If list contains wildcard -> return known provider models """ - unique_models: Final = [] - - def append_unique(models): - for model in models: - if model not in unique_models and model != SpecialModelNames.no_default_models.value: - unique_models.append(model) - - if key_models: - append_unique(key_models) - elif team_models: - append_unique(team_models) - else: - append_unique(proxy_model_list) - if include_model_access_groups: - append_unique(list(model_access_groups.keys())) # TODO: keys order - - if user_model: - append_unique([user_model]) - - if infer_model_from_keys: - valid_models: Final = get_valid_models() - append_unique(valid_models) - if only_model_access_groups: - model_access_groups_to_return: Final[list[str]] = [] - for model in unique_models: - if model in model_access_groups: - model_access_groups_to_return.append(model) - return model_access_groups_to_return - - all_wildcard_models: Final = _get_wildcard_models( - unique_models=unique_models, - return_wildcard_routes=return_wildcard_routes, - llm_router=llm_router, - team_id=team_id, + return [ + model + for model in _unexpanded_model_list( + key_models=key_models, + team_models=team_models, + proxy_model_list=proxy_model_list, + user_model=user_model, + infer_model_from_keys=infer_model_from_keys, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + ) + if model in model_access_groups + ] + return list( + iter_complete_model_list( + key_models=key_models, + team_models=team_models, + proxy_model_list=proxy_model_list, + user_model=user_model, + infer_model_from_keys=infer_model_from_keys, + return_wildcard_routes=return_wildcard_routes, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + team_id=team_id, + ) ) - complete_model_list: Final = unique_models + all_wildcard_models - return complete_model_list +def _unexpanded_model_list( + key_models: Sequence[str], + team_models: Sequence[str], + proxy_model_list: Sequence[str], + user_model: str | None, + infer_model_from_keys: bool | None, + model_access_groups: Mapping[str, Sequence[str]], + include_model_access_groups: bool | None, +) -> list[str]: + granted_model_names: Final = ( + key_models + or team_models + or ( + *proxy_model_list, + *(model_access_groups if include_model_access_groups else ()), + *((user_model,) if user_model else ()), + *(get_valid_models() if infer_model_from_keys else ()), + ) + ) + return [model for model in dict.fromkeys(granted_model_names) if model != SpecialModelNames.no_default_models.value] + + +def iter_complete_model_list( + key_models: Sequence[str], + team_models: Sequence[str], + proxy_model_list: Sequence[str], + user_model: str | None, + infer_model_from_keys: bool | None, + return_wildcard_routes: bool | None, + llm_router: Router | None, + model_access_groups: Mapping[str, Sequence[str]], + include_model_access_groups: bool | None, + team_id: str | None, +) -> Iterator[str]: + unexpanded_models: Final = _unexpanded_model_list( + key_models=key_models, + team_models=team_models, + proxy_model_list=proxy_model_list, + user_model=user_model, + infer_model_from_keys=infer_model_from_keys, + model_access_groups=model_access_groups, + include_model_access_groups=include_model_access_groups, + ) + yield from (model for model in unexpanded_models if not _check_wildcard_routing(model=model)) + routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) + for wildcard_route in (model for model in unexpanded_models if _check_wildcard_routing(model=model)): + if return_wildcard_routes: + yield wildcard_route + yield from _wildcard_route_models(wildcard_route, llm_router, team_id, routes_to_model) def _hydrate_litellm_credential_name( @@ -558,36 +597,22 @@ def expand_wildcard_deployments_for_model_info( return expanded -def _get_wildcard_models( - unique_models: list[str], - return_wildcard_routes: bool | None = False, - llm_router: Router | None = None, - team_id: str | None = None, -) -> list[str]: - wildcard_routes: Final = tuple(model for model in unique_models if _check_wildcard_routing(model=model)) - routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) - all_wildcard_models: Final = [] - for model in wildcard_routes: - if return_wildcard_routes: - all_wildcard_models.append(model) - - model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None - if model_list: - for router_model in model_list: - all_wildcard_models.extend( - get_known_models_from_wildcard( - wildcard_model=model, - litellm_params=LiteLLM_Params(**router_model["litellm_params"]), - routes_to_model=routes_to_model, - ) - ) - else: - all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None)) - - for model in frozenset(wildcard_routes): - unique_models.remove(model) - - return all_wildcard_models +def _wildcard_route_models( + wildcard_route: str, + llm_router: Router | None, + team_id: str | None, + routes_to_model: Callable[[str, str], bool], +) -> Iterator[str]: + router_models: Final = llm_router.get_model_list(model_name=wildcard_route, team_id=team_id) if llm_router else None + if not router_models: + yield from get_known_models_from_wildcard(wildcard_model=wildcard_route, litellm_params=None) + return + for router_model in router_models: + yield from get_known_models_from_wildcard( + wildcard_model=wildcard_route, + litellm_params=LiteLLM_Params(**router_model["litellm_params"]), + routes_to_model=routes_to_model, + ) def get_all_fallbacks( diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 907fd11347c..0073528c61b 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -559,7 +559,7 @@ def get_team_provider_credentials( return None from litellm.proxy._types import SpecialModelNames - from litellm.proxy.auth.model_checks import get_complete_model_list, get_key_models + from litellm.proxy.auth.model_checks import get_key_models, iter_complete_model_list team_id: Final = user_api_key_dict.team_id team_models: Final = user_api_key_dict.team_models or [] @@ -637,35 +637,29 @@ def get_team_provider_credentials( # effective allowlist (sentinels and access groups already expanded by # get_key_models) wins when set; otherwise the team's allowlist applies. # The all-proxy-models sentinel isn't expanded by - # get_complete_model_list, so normalize it to an empty allowlist, which + # iter_complete_model_list, so normalize it to an empty allowlist, which # defers to the team-scoped proxy model list. A team or key with a # restricted allowlist (e.g. anthropic only) therefore never resolves # another provider's key. grants_all_models: Final = SpecialModelNames.all_proxy_models.value in team_models effective_team_models: Final = [] if grants_all_models else team_models - models_to_try: Final = list( - dict.fromkeys( - get_complete_model_list( - key_models=list(key_model_allowlist), - team_models=effective_team_models, - proxy_model_list=proxy_model_list, - user_model=None, - infer_model_from_keys=False, - return_wildcard_routes=True, - llm_router=llm_router, - model_access_groups=model_access_groups, - include_model_access_groups=True, - team_id=team_id, - ) - ) + models_to_try: Final = iter_complete_model_list( + key_models=key_model_allowlist, + team_models=effective_team_models, + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + return_wildcard_routes=True, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=True, + team_id=team_id, + ) + return next( + (credentials for credentials in map(_provider_credentials, models_to_try) if credentials is not None), + None, ) - for model_name in models_to_try: - credentials = _provider_credentials(model_name) - if credentials is not None: - return credentials - - return None HTTP_HEADER_VALUE: Final = re.compile(r"[\x21-\x7e]+(?:[ \t]+[\x21-\x7e]+)*") diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index dc86253e49a..5f2f4c465e7 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -806,7 +806,7 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve ] ), user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), - custom_llm_provider="hosted_vllm", + custom_llm_provider="openai", ) ) assert await asyncio.to_thread(listing_started.wait, 3) @@ -814,7 +814,46 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve await lookup assert listing_answered.is_set() - assert data.get("api_base") == "http://vllm-credential-lookup.test:8000/v1" + assert data == {} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "custom_llm_provider, vllm_api_base, applied_credential", + [ + ("openai", "http://vllm-beside-openai.test:8000/v1", ("api_key", "openai-key")), + ("hosted_vllm", "http://vllm-route-match.test:8000/v1", ("api_base", "http://vllm-route-match.test:8000/v1")), + ], +) +async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wildcard_server( + monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, vllm_api_base: str, applied_credential: tuple[str, str] +) -> None: + import litellm + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + data: Final[dict[str, object]] = {} + + with respx.mock(assert_all_called=False) as vllm_server: + listing: Final = vllm_server.get(f"{vllm_api_base}/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]}) + ) + await apply_team_provider_credentials( + data=data, + llm_router=Router( + model_list=[ + { + "model_name": "hosted_vllm/*", + "litellm_params": {"model": "hosted_vllm/*", "api_base": vllm_api_base}, + }, + {"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "openai-key"}}, + ] + ), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + custom_llm_provider=custom_llm_provider, + ) + + assert not listing.called + assert data.get(applied_credential[0]) == applied_credential[1] def _apply_team_a_openai_credentials(data: dict[str, object]) -> None: