diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index a997b53d265..7739cd9941e 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -255,7 +255,7 @@ def get_complete_model_list( if model in model_access_groups ] return list( - iter_complete_model_list( + _iter_complete_model_list( key_models=key_models, team_models=team_models, proxy_model_list=proxy_model_list, @@ -292,7 +292,7 @@ def _unexpanded_model_list( return [model for model in dict.fromkeys(granted_model_names) if model != SpecialModelNames.no_default_models.value] -def iter_complete_model_list( +def _iter_complete_model_list( key_models: Sequence[str], team_models: Sequence[str], proxy_model_list: Sequence[str], @@ -321,6 +321,31 @@ def iter_complete_model_list( yield from _wildcard_route_models(wildcard_route, llm_router, team_id, routes_to_model) +def iter_model_names_expanding_wildcards_last( + key_models: Sequence[str], + team_models: Sequence[str], + proxy_model_list: Sequence[str], + llm_router: Router | None, + model_access_groups: Mapping[str, Sequence[str]], + team_id: str | None, +) -> Iterator[str]: + unexpanded_models: Final = _unexpanded_model_list( + key_models=key_models, + team_models=team_models, + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + model_access_groups=model_access_groups, + include_model_access_groups=True, + ) + wildcard_routes: Final = tuple(model for model in unexpanded_models if _check_wildcard_routing(model=model)) + yield from (model for model in unexpanded_models if not _check_wildcard_routing(model=model)) + yield from wildcard_routes + routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) + for wildcard_route in wildcard_routes: + yield from _wildcard_route_models(wildcard_route, llm_router, team_id, routes_to_model) + + def _hydrate_litellm_credential_name( litellm_params: LiteLLM_Params | None, ) -> LiteLLM_Params | None: diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 0073528c61b..371456f3991 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -559,7 +559,7 @@ def get_team_provider_credentials( return None from litellm.proxy._types import SpecialModelNames - from litellm.proxy.auth.model_checks import get_key_models, iter_complete_model_list + from litellm.proxy.auth.model_checks import get_key_models, iter_model_names_expanding_wildcards_last team_id: Final = user_api_key_dict.team_id team_models: Final = user_api_key_dict.team_models or [] @@ -637,23 +637,19 @@ def get_team_provider_credentials( # effective allowlist (sentinels and access groups already expanded by # get_key_models) wins when set; otherwise the team's allowlist applies. # The all-proxy-models sentinel isn't expanded by - # iter_complete_model_list, so normalize it to an empty allowlist, which + # iter_model_names_expanding_wildcards_last, so normalize it to an empty allowlist, which # defers to the team-scoped proxy model list. A team or key with a # restricted allowlist (e.g. anthropic only) therefore never resolves # another provider's key. grants_all_models: Final = SpecialModelNames.all_proxy_models.value in team_models effective_team_models: Final = [] if grants_all_models else team_models - models_to_try: Final = iter_complete_model_list( + models_to_try: Final = iter_model_names_expanding_wildcards_last( key_models=key_model_allowlist, team_models=effective_team_models, proxy_model_list=proxy_model_list, - user_model=None, - infer_model_from_keys=False, - return_wildcard_routes=True, llm_router=llm_router, model_access_groups=model_access_groups, - include_model_access_groups=True, team_id=team_id, ) return next( diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index 5f2f4c465e7..6d1fb00308f 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -819,14 +819,29 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve @pytest.mark.asyncio @pytest.mark.parametrize( - "custom_llm_provider, vllm_api_base, applied_credential", + "custom_llm_provider, vllm_api_base, openai_deployment, applied_credential", [ - ("openai", "http://vllm-beside-openai.test:8000/v1", ("api_key", "openai-key")), - ("hosted_vllm", "http://vllm-route-match.test:8000/v1", ("api_base", "http://vllm-route-match.test:8000/v1")), + ("openai", "http://vllm-beside-openai.test:8000/v1", ("gpt-4o", "openai/gpt-4o"), ("api_key", "openai-key")), + ( + "openai", + "http://vllm-before-openai-wildcard.test:8000/v1", + ("openai/*", "openai/*"), + ("api_key", "openai-key"), + ), + ( + "hosted_vllm", + "http://vllm-route-match.test:8000/v1", + ("gpt-4o", "openai/gpt-4o"), + ("api_base", "http://vllm-route-match.test:8000/v1"), + ), ], ) async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wildcard_server( - monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, vllm_api_base: str, applied_credential: tuple[str, str] + monkeypatch: pytest.MonkeyPatch, + custom_llm_provider: str, + vllm_api_base: str, + openai_deployment: tuple[str, str], + applied_credential: tuple[str, str], ) -> None: import litellm @@ -845,7 +860,10 @@ async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wi "model_name": "hosted_vllm/*", "litellm_params": {"model": "hosted_vllm/*", "api_base": vllm_api_base}, }, - {"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "openai-key"}}, + { + "model_name": openai_deployment[0], + "litellm_params": {"model": openai_deployment[1], "api_key": "openai-key"}, + }, ] ), user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),