fix(proxy): stop the file credential lookup at the first match before listing wildcard models

The fallback lookup now walks the allowed model names lazily, in the same
order as before, so a hung vLLM server is listed only when no configured
name serves the requested provider. The vLLM auth headers no longer
mutate the caller's headers dict.
This commit is contained in:
mateo-berri 2026-10-29 07:01:10 -07:00
parent 22e417363a
commit 1f03ebc983
4 changed files with 151 additions and 94 deletions

View file

@ -56,8 +56,7 @@ class VLLMModelInfo(BaseLLMModelInfo):
api_key: str | None = None,
api_base: str | None = None,
) -> dict:
headers.update(_auth_headers(api_key))
return headers
return {**headers, **_auth_headers(api_key)}
@staticmethod
def get_api_base(api_base: str | None = None) -> str:

View file

@ -6,7 +6,7 @@ import copy
import functools
import itertools
import re
from collections.abc import Callable, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from concurrent.futures import ThreadPoolExecutor
from typing import Final, ParamSpec, TypeVar
@ -240,46 +240,85 @@ def get_complete_model_list(
If list contains wildcard -> return known provider models
"""
unique_models: Final = []
def append_unique(models):
for model in models:
if model not in unique_models and model != SpecialModelNames.no_default_models.value:
unique_models.append(model)
if key_models:
append_unique(key_models)
elif team_models:
append_unique(team_models)
else:
append_unique(proxy_model_list)
if include_model_access_groups:
append_unique(list(model_access_groups.keys())) # TODO: keys order
if user_model:
append_unique([user_model])
if infer_model_from_keys:
valid_models: Final = get_valid_models()
append_unique(valid_models)
if only_model_access_groups:
model_access_groups_to_return: Final[list[str]] = []
for model in unique_models:
if model in model_access_groups:
model_access_groups_to_return.append(model)
return model_access_groups_to_return
all_wildcard_models: Final = _get_wildcard_models(
unique_models=unique_models,
return_wildcard_routes=return_wildcard_routes,
llm_router=llm_router,
team_id=team_id,
return [
model
for model in _unexpanded_model_list(
key_models=key_models,
team_models=team_models,
proxy_model_list=proxy_model_list,
user_model=user_model,
infer_model_from_keys=infer_model_from_keys,
model_access_groups=model_access_groups,
include_model_access_groups=include_model_access_groups,
)
if model in model_access_groups
]
return list(
iter_complete_model_list(
key_models=key_models,
team_models=team_models,
proxy_model_list=proxy_model_list,
user_model=user_model,
infer_model_from_keys=infer_model_from_keys,
return_wildcard_routes=return_wildcard_routes,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=include_model_access_groups,
team_id=team_id,
)
)
complete_model_list: Final = unique_models + all_wildcard_models
return complete_model_list
def _unexpanded_model_list(
key_models: Sequence[str],
team_models: Sequence[str],
proxy_model_list: Sequence[str],
user_model: str | None,
infer_model_from_keys: bool | None,
model_access_groups: Mapping[str, Sequence[str]],
include_model_access_groups: bool | None,
) -> list[str]:
granted_model_names: Final = (
key_models
or team_models
or (
*proxy_model_list,
*(model_access_groups if include_model_access_groups else ()),
*((user_model,) if user_model else ()),
*(get_valid_models() if infer_model_from_keys else ()),
)
)
return [model for model in dict.fromkeys(granted_model_names) if model != SpecialModelNames.no_default_models.value]
def iter_complete_model_list(
key_models: Sequence[str],
team_models: Sequence[str],
proxy_model_list: Sequence[str],
user_model: str | None,
infer_model_from_keys: bool | None,
return_wildcard_routes: bool | None,
llm_router: Router | None,
model_access_groups: Mapping[str, Sequence[str]],
include_model_access_groups: bool | None,
team_id: str | None,
) -> Iterator[str]:
unexpanded_models: Final = _unexpanded_model_list(
key_models=key_models,
team_models=team_models,
proxy_model_list=proxy_model_list,
user_model=user_model,
infer_model_from_keys=infer_model_from_keys,
model_access_groups=model_access_groups,
include_model_access_groups=include_model_access_groups,
)
yield from (model for model in unexpanded_models if not _check_wildcard_routing(model=model))
routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id)
for wildcard_route in (model for model in unexpanded_models if _check_wildcard_routing(model=model)):
if return_wildcard_routes:
yield wildcard_route
yield from _wildcard_route_models(wildcard_route, llm_router, team_id, routes_to_model)
def _hydrate_litellm_credential_name(
@ -558,36 +597,22 @@ def expand_wildcard_deployments_for_model_info(
return expanded
def _get_wildcard_models(
unique_models: list[str],
return_wildcard_routes: bool | None = False,
llm_router: Router | None = None,
team_id: str | None = None,
) -> list[str]:
wildcard_routes: Final = tuple(model for model in unique_models if _check_wildcard_routing(model=model))
routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id)
all_wildcard_models: Final = []
for model in wildcard_routes:
if return_wildcard_routes:
all_wildcard_models.append(model)
model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None
if model_list:
for router_model in model_list:
all_wildcard_models.extend(
get_known_models_from_wildcard(
wildcard_model=model,
litellm_params=LiteLLM_Params(**router_model["litellm_params"]),
routes_to_model=routes_to_model,
)
)
else:
all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None))
for model in frozenset(wildcard_routes):
unique_models.remove(model)
return all_wildcard_models
def _wildcard_route_models(
wildcard_route: str,
llm_router: Router | None,
team_id: str | None,
routes_to_model: Callable[[str, str], bool],
) -> Iterator[str]:
router_models: Final = llm_router.get_model_list(model_name=wildcard_route, team_id=team_id) if llm_router else None
if not router_models:
yield from get_known_models_from_wildcard(wildcard_model=wildcard_route, litellm_params=None)
return
for router_model in router_models:
yield from get_known_models_from_wildcard(
wildcard_model=wildcard_route,
litellm_params=LiteLLM_Params(**router_model["litellm_params"]),
routes_to_model=routes_to_model,
)
def get_all_fallbacks(

View file

@ -559,7 +559,7 @@ def get_team_provider_credentials(
return None
from litellm.proxy._types import SpecialModelNames
from litellm.proxy.auth.model_checks import get_complete_model_list, get_key_models
from litellm.proxy.auth.model_checks import get_key_models, iter_complete_model_list
team_id: Final = user_api_key_dict.team_id
team_models: Final = user_api_key_dict.team_models or []
@ -637,35 +637,29 @@ def get_team_provider_credentials(
# effective allowlist (sentinels and access groups already expanded by
# get_key_models) wins when set; otherwise the team's allowlist applies.
# The all-proxy-models sentinel isn't expanded by
# get_complete_model_list, so normalize it to an empty allowlist, which
# iter_complete_model_list, so normalize it to an empty allowlist, which
# defers to the team-scoped proxy model list. A team or key with a
# restricted allowlist (e.g. anthropic only) therefore never resolves
# another provider's key.
grants_all_models: Final = SpecialModelNames.all_proxy_models.value in team_models
effective_team_models: Final = [] if grants_all_models else team_models
models_to_try: Final = list(
dict.fromkeys(
get_complete_model_list(
key_models=list(key_model_allowlist),
team_models=effective_team_models,
proxy_model_list=proxy_model_list,
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=True,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=True,
team_id=team_id,
)
)
models_to_try: Final = iter_complete_model_list(
key_models=key_model_allowlist,
team_models=effective_team_models,
proxy_model_list=proxy_model_list,
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=True,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=True,
team_id=team_id,
)
return next(
(credentials for credentials in map(_provider_credentials, models_to_try) if credentials is not None),
None,
)
for model_name in models_to_try:
credentials = _provider_credentials(model_name)
if credentials is not None:
return credentials
return None
HTTP_HEADER_VALUE: Final = re.compile(r"[\x21-\x7e]+(?:[ \t]+[\x21-\x7e]+)*")

View file

@ -806,7 +806,7 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve
]
),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
custom_llm_provider="hosted_vllm",
custom_llm_provider="openai",
)
)
assert await asyncio.to_thread(listing_started.wait, 3)
@ -814,7 +814,46 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve
await lookup
assert listing_answered.is_set()
assert data.get("api_base") == "http://vllm-credential-lookup.test:8000/v1"
assert data == {}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"custom_llm_provider, vllm_api_base, applied_credential",
[
("openai", "http://vllm-beside-openai.test:8000/v1", ("api_key", "openai-key")),
("hosted_vllm", "http://vllm-route-match.test:8000/v1", ("api_base", "http://vllm-route-match.test:8000/v1")),
],
)
async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wildcard_server(
monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, vllm_api_base: str, applied_credential: tuple[str, str]
) -> None:
import litellm
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
data: Final[dict[str, object]] = {}
with respx.mock(assert_all_called=False) as vllm_server:
listing: Final = vllm_server.get(f"{vllm_api_base}/models").mock(
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
)
await apply_team_provider_credentials(
data=data,
llm_router=Router(
model_list=[
{
"model_name": "hosted_vllm/*",
"litellm_params": {"model": "hosted_vllm/*", "api_base": vllm_api_base},
},
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "openai-key"}},
]
),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
custom_llm_provider=custom_llm_provider,
)
assert not listing.called
assert data.get(applied_credential[0]) == applied_credential[1]
def _apply_team_a_openai_credentials(data: dict[str, object]) -> None: