fix(proxy): try every configured wildcard route before listing any wildcard's models in the file credential lookup

This commit is contained in:
mateo-berri 2026-10-29 07:33:57 -07:00
parent 1f03ebc983
commit 83d52c7646
3 changed files with 53 additions and 14 deletions

View file

@ -255,7 +255,7 @@ def get_complete_model_list(
if model in model_access_groups
]
return list(
iter_complete_model_list(
_iter_complete_model_list(
key_models=key_models,
team_models=team_models,
proxy_model_list=proxy_model_list,
@ -292,7 +292,7 @@ def _unexpanded_model_list(
return [model for model in dict.fromkeys(granted_model_names) if model != SpecialModelNames.no_default_models.value]
def iter_complete_model_list(
def _iter_complete_model_list(
key_models: Sequence[str],
team_models: Sequence[str],
proxy_model_list: Sequence[str],
@ -321,6 +321,31 @@ def iter_complete_model_list(
yield from _wildcard_route_models(wildcard_route, llm_router, team_id, routes_to_model)
def iter_model_names_expanding_wildcards_last(
key_models: Sequence[str],
team_models: Sequence[str],
proxy_model_list: Sequence[str],
llm_router: Router | None,
model_access_groups: Mapping[str, Sequence[str]],
team_id: str | None,
) -> Iterator[str]:
unexpanded_models: Final = _unexpanded_model_list(
key_models=key_models,
team_models=team_models,
proxy_model_list=proxy_model_list,
user_model=None,
infer_model_from_keys=False,
model_access_groups=model_access_groups,
include_model_access_groups=True,
)
wildcard_routes: Final = tuple(model for model in unexpanded_models if _check_wildcard_routing(model=model))
yield from (model for model in unexpanded_models if not _check_wildcard_routing(model=model))
yield from wildcard_routes
routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id)
for wildcard_route in wildcard_routes:
yield from _wildcard_route_models(wildcard_route, llm_router, team_id, routes_to_model)
def _hydrate_litellm_credential_name(
litellm_params: LiteLLM_Params | None,
) -> LiteLLM_Params | None:

View file

@ -559,7 +559,7 @@ def get_team_provider_credentials(
return None
from litellm.proxy._types import SpecialModelNames
from litellm.proxy.auth.model_checks import get_key_models, iter_complete_model_list
from litellm.proxy.auth.model_checks import get_key_models, iter_model_names_expanding_wildcards_last
team_id: Final = user_api_key_dict.team_id
team_models: Final = user_api_key_dict.team_models or []
@ -637,23 +637,19 @@ def get_team_provider_credentials(
# effective allowlist (sentinels and access groups already expanded by
# get_key_models) wins when set; otherwise the team's allowlist applies.
# The all-proxy-models sentinel isn't expanded by
# iter_complete_model_list, so normalize it to an empty allowlist, which
# iter_model_names_expanding_wildcards_last, so normalize it to an empty allowlist, which
# defers to the team-scoped proxy model list. A team or key with a
# restricted allowlist (e.g. anthropic only) therefore never resolves
# another provider's key.
grants_all_models: Final = SpecialModelNames.all_proxy_models.value in team_models
effective_team_models: Final = [] if grants_all_models else team_models
models_to_try: Final = iter_complete_model_list(
models_to_try: Final = iter_model_names_expanding_wildcards_last(
key_models=key_model_allowlist,
team_models=effective_team_models,
proxy_model_list=proxy_model_list,
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=True,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=True,
team_id=team_id,
)
return next(

View file

@ -819,14 +819,29 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve
@pytest.mark.asyncio
@pytest.mark.parametrize(
"custom_llm_provider, vllm_api_base, applied_credential",
"custom_llm_provider, vllm_api_base, openai_deployment, applied_credential",
[
("openai", "http://vllm-beside-openai.test:8000/v1", ("api_key", "openai-key")),
("hosted_vllm", "http://vllm-route-match.test:8000/v1", ("api_base", "http://vllm-route-match.test:8000/v1")),
("openai", "http://vllm-beside-openai.test:8000/v1", ("gpt-4o", "openai/gpt-4o"), ("api_key", "openai-key")),
(
"openai",
"http://vllm-before-openai-wildcard.test:8000/v1",
("openai/*", "openai/*"),
("api_key", "openai-key"),
),
(
"hosted_vllm",
"http://vllm-route-match.test:8000/v1",
("gpt-4o", "openai/gpt-4o"),
("api_base", "http://vllm-route-match.test:8000/v1"),
),
],
)
async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wildcard_server(
monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, vllm_api_base: str, applied_credential: tuple[str, str]
monkeypatch: pytest.MonkeyPatch,
custom_llm_provider: str,
vllm_api_base: str,
openai_deployment: tuple[str, str],
applied_credential: tuple[str, str],
) -> None:
import litellm
@ -845,7 +860,10 @@ async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wi
"model_name": "hosted_vllm/*",
"litellm_params": {"model": "hosted_vllm/*", "api_base": vllm_api_base},
},
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "openai-key"}},
{
"model_name": openai_deployment[0],
"litellm_params": {"model": openai_deployment[1], "api_key": "openai-key"},
},
]
),
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),