mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): try every configured wildcard route before listing any wildcard's models in the file credential lookup
This commit is contained in:
parent
1f03ebc983
commit
83d52c7646
3 changed files with 53 additions and 14 deletions
|
|
@ -255,7 +255,7 @@ def get_complete_model_list(
|
|||
if model in model_access_groups
|
||||
]
|
||||
return list(
|
||||
iter_complete_model_list(
|
||||
_iter_complete_model_list(
|
||||
key_models=key_models,
|
||||
team_models=team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
|
|
@ -292,7 +292,7 @@ def _unexpanded_model_list(
|
|||
return [model for model in dict.fromkeys(granted_model_names) if model != SpecialModelNames.no_default_models.value]
|
||||
|
||||
|
||||
def iter_complete_model_list(
|
||||
def _iter_complete_model_list(
|
||||
key_models: Sequence[str],
|
||||
team_models: Sequence[str],
|
||||
proxy_model_list: Sequence[str],
|
||||
|
|
@ -321,6 +321,31 @@ def iter_complete_model_list(
|
|||
yield from _wildcard_route_models(wildcard_route, llm_router, team_id, routes_to_model)
|
||||
|
||||
|
||||
def iter_model_names_expanding_wildcards_last(
|
||||
key_models: Sequence[str],
|
||||
team_models: Sequence[str],
|
||||
proxy_model_list: Sequence[str],
|
||||
llm_router: Router | None,
|
||||
model_access_groups: Mapping[str, Sequence[str]],
|
||||
team_id: str | None,
|
||||
) -> Iterator[str]:
|
||||
unexpanded_models: Final = _unexpanded_model_list(
|
||||
key_models=key_models,
|
||||
team_models=team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=True,
|
||||
)
|
||||
wildcard_routes: Final = tuple(model for model in unexpanded_models if _check_wildcard_routing(model=model))
|
||||
yield from (model for model in unexpanded_models if not _check_wildcard_routing(model=model))
|
||||
yield from wildcard_routes
|
||||
routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id)
|
||||
for wildcard_route in wildcard_routes:
|
||||
yield from _wildcard_route_models(wildcard_route, llm_router, team_id, routes_to_model)
|
||||
|
||||
|
||||
def _hydrate_litellm_credential_name(
|
||||
litellm_params: LiteLLM_Params | None,
|
||||
) -> LiteLLM_Params | None:
|
||||
|
|
|
|||
|
|
@ -559,7 +559,7 @@ def get_team_provider_credentials(
|
|||
return None
|
||||
|
||||
from litellm.proxy._types import SpecialModelNames
|
||||
from litellm.proxy.auth.model_checks import get_key_models, iter_complete_model_list
|
||||
from litellm.proxy.auth.model_checks import get_key_models, iter_model_names_expanding_wildcards_last
|
||||
|
||||
team_id: Final = user_api_key_dict.team_id
|
||||
team_models: Final = user_api_key_dict.team_models or []
|
||||
|
|
@ -637,23 +637,19 @@ def get_team_provider_credentials(
|
|||
# effective allowlist (sentinels and access groups already expanded by
|
||||
# get_key_models) wins when set; otherwise the team's allowlist applies.
|
||||
# The all-proxy-models sentinel isn't expanded by
|
||||
# iter_complete_model_list, so normalize it to an empty allowlist, which
|
||||
# iter_model_names_expanding_wildcards_last, so normalize it to an empty allowlist, which
|
||||
# defers to the team-scoped proxy model list. A team or key with a
|
||||
# restricted allowlist (e.g. anthropic only) therefore never resolves
|
||||
# another provider's key.
|
||||
grants_all_models: Final = SpecialModelNames.all_proxy_models.value in team_models
|
||||
effective_team_models: Final = [] if grants_all_models else team_models
|
||||
|
||||
models_to_try: Final = iter_complete_model_list(
|
||||
models_to_try: Final = iter_model_names_expanding_wildcards_last(
|
||||
key_models=key_model_allowlist,
|
||||
team_models=effective_team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
return_wildcard_routes=True,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=True,
|
||||
team_id=team_id,
|
||||
)
|
||||
return next(
|
||||
|
|
|
|||
|
|
@ -819,14 +819,29 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve
|
|||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"custom_llm_provider, vllm_api_base, applied_credential",
|
||||
"custom_llm_provider, vllm_api_base, openai_deployment, applied_credential",
|
||||
[
|
||||
("openai", "http://vllm-beside-openai.test:8000/v1", ("api_key", "openai-key")),
|
||||
("hosted_vllm", "http://vllm-route-match.test:8000/v1", ("api_base", "http://vllm-route-match.test:8000/v1")),
|
||||
("openai", "http://vllm-beside-openai.test:8000/v1", ("gpt-4o", "openai/gpt-4o"), ("api_key", "openai-key")),
|
||||
(
|
||||
"openai",
|
||||
"http://vllm-before-openai-wildcard.test:8000/v1",
|
||||
("openai/*", "openai/*"),
|
||||
("api_key", "openai-key"),
|
||||
),
|
||||
(
|
||||
"hosted_vllm",
|
||||
"http://vllm-route-match.test:8000/v1",
|
||||
("gpt-4o", "openai/gpt-4o"),
|
||||
("api_base", "http://vllm-route-match.test:8000/v1"),
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wildcard_server(
|
||||
monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, vllm_api_base: str, applied_credential: tuple[str, str]
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
custom_llm_provider: str,
|
||||
vllm_api_base: str,
|
||||
openai_deployment: tuple[str, str],
|
||||
applied_credential: tuple[str, str],
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
|
|
@ -845,7 +860,10 @@ async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wi
|
|||
"model_name": "hosted_vllm/*",
|
||||
"litellm_params": {"model": "hosted_vllm/*", "api_base": vllm_api_base},
|
||||
},
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "openai-key"}},
|
||||
{
|
||||
"model_name": openai_deployment[0],
|
||||
"litellm_params": {"model": openai_deployment[1], "api_key": "openai-key"},
|
||||
},
|
||||
]
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue