mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): stop the file credential lookup at the first match before listing wildcard models
The fallback lookup now walks the allowed model names lazily, in the same order as before, so a hung vLLM server is listed only when no configured name serves the requested provider. The vLLM auth headers no longer mutate the caller's headers dict.
This commit is contained in:
parent
22e417363a
commit
1f03ebc983
4 changed files with 151 additions and 94 deletions
|
|
@ -56,8 +56,7 @@ class VLLMModelInfo(BaseLLMModelInfo):
|
|||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict:
|
||||
headers.update(_auth_headers(api_key))
|
||||
return headers
|
||||
return {**headers, **_auth_headers(api_key)}
|
||||
|
||||
@staticmethod
|
||||
def get_api_base(api_base: str | None = None) -> str:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import copy
|
|||
import functools
|
||||
import itertools
|
||||
import re
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import Final, ParamSpec, TypeVar
|
||||
|
||||
|
|
@ -240,46 +240,85 @@ def get_complete_model_list(
|
|||
If list contains wildcard -> return known provider models
|
||||
"""
|
||||
|
||||
unique_models: Final = []
|
||||
|
||||
def append_unique(models):
|
||||
for model in models:
|
||||
if model not in unique_models and model != SpecialModelNames.no_default_models.value:
|
||||
unique_models.append(model)
|
||||
|
||||
if key_models:
|
||||
append_unique(key_models)
|
||||
elif team_models:
|
||||
append_unique(team_models)
|
||||
else:
|
||||
append_unique(proxy_model_list)
|
||||
if include_model_access_groups:
|
||||
append_unique(list(model_access_groups.keys())) # TODO: keys order
|
||||
|
||||
if user_model:
|
||||
append_unique([user_model])
|
||||
|
||||
if infer_model_from_keys:
|
||||
valid_models: Final = get_valid_models()
|
||||
append_unique(valid_models)
|
||||
|
||||
if only_model_access_groups:
|
||||
model_access_groups_to_return: Final[list[str]] = []
|
||||
for model in unique_models:
|
||||
if model in model_access_groups:
|
||||
model_access_groups_to_return.append(model)
|
||||
return model_access_groups_to_return
|
||||
|
||||
all_wildcard_models: Final = _get_wildcard_models(
|
||||
unique_models=unique_models,
|
||||
return_wildcard_routes=return_wildcard_routes,
|
||||
llm_router=llm_router,
|
||||
team_id=team_id,
|
||||
return [
|
||||
model
|
||||
for model in _unexpanded_model_list(
|
||||
key_models=key_models,
|
||||
team_models=team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=user_model,
|
||||
infer_model_from_keys=infer_model_from_keys,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups,
|
||||
)
|
||||
if model in model_access_groups
|
||||
]
|
||||
return list(
|
||||
iter_complete_model_list(
|
||||
key_models=key_models,
|
||||
team_models=team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=user_model,
|
||||
infer_model_from_keys=infer_model_from_keys,
|
||||
return_wildcard_routes=return_wildcard_routes,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups,
|
||||
team_id=team_id,
|
||||
)
|
||||
)
|
||||
|
||||
complete_model_list: Final = unique_models + all_wildcard_models
|
||||
|
||||
return complete_model_list
|
||||
def _unexpanded_model_list(
|
||||
key_models: Sequence[str],
|
||||
team_models: Sequence[str],
|
||||
proxy_model_list: Sequence[str],
|
||||
user_model: str | None,
|
||||
infer_model_from_keys: bool | None,
|
||||
model_access_groups: Mapping[str, Sequence[str]],
|
||||
include_model_access_groups: bool | None,
|
||||
) -> list[str]:
|
||||
granted_model_names: Final = (
|
||||
key_models
|
||||
or team_models
|
||||
or (
|
||||
*proxy_model_list,
|
||||
*(model_access_groups if include_model_access_groups else ()),
|
||||
*((user_model,) if user_model else ()),
|
||||
*(get_valid_models() if infer_model_from_keys else ()),
|
||||
)
|
||||
)
|
||||
return [model for model in dict.fromkeys(granted_model_names) if model != SpecialModelNames.no_default_models.value]
|
||||
|
||||
|
||||
def iter_complete_model_list(
|
||||
key_models: Sequence[str],
|
||||
team_models: Sequence[str],
|
||||
proxy_model_list: Sequence[str],
|
||||
user_model: str | None,
|
||||
infer_model_from_keys: bool | None,
|
||||
return_wildcard_routes: bool | None,
|
||||
llm_router: Router | None,
|
||||
model_access_groups: Mapping[str, Sequence[str]],
|
||||
include_model_access_groups: bool | None,
|
||||
team_id: str | None,
|
||||
) -> Iterator[str]:
|
||||
unexpanded_models: Final = _unexpanded_model_list(
|
||||
key_models=key_models,
|
||||
team_models=team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=user_model,
|
||||
infer_model_from_keys=infer_model_from_keys,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=include_model_access_groups,
|
||||
)
|
||||
yield from (model for model in unexpanded_models if not _check_wildcard_routing(model=model))
|
||||
routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id)
|
||||
for wildcard_route in (model for model in unexpanded_models if _check_wildcard_routing(model=model)):
|
||||
if return_wildcard_routes:
|
||||
yield wildcard_route
|
||||
yield from _wildcard_route_models(wildcard_route, llm_router, team_id, routes_to_model)
|
||||
|
||||
|
||||
def _hydrate_litellm_credential_name(
|
||||
|
|
@ -558,36 +597,22 @@ def expand_wildcard_deployments_for_model_info(
|
|||
return expanded
|
||||
|
||||
|
||||
def _get_wildcard_models(
|
||||
unique_models: list[str],
|
||||
return_wildcard_routes: bool | None = False,
|
||||
llm_router: Router | None = None,
|
||||
team_id: str | None = None,
|
||||
) -> list[str]:
|
||||
wildcard_routes: Final = tuple(model for model in unique_models if _check_wildcard_routing(model=model))
|
||||
routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id)
|
||||
all_wildcard_models: Final = []
|
||||
for model in wildcard_routes:
|
||||
if return_wildcard_routes:
|
||||
all_wildcard_models.append(model)
|
||||
|
||||
model_list = llm_router.get_model_list(model_name=model, team_id=team_id) if llm_router else None
|
||||
if model_list:
|
||||
for router_model in model_list:
|
||||
all_wildcard_models.extend(
|
||||
get_known_models_from_wildcard(
|
||||
wildcard_model=model,
|
||||
litellm_params=LiteLLM_Params(**router_model["litellm_params"]),
|
||||
routes_to_model=routes_to_model,
|
||||
)
|
||||
)
|
||||
else:
|
||||
all_wildcard_models.extend(get_known_models_from_wildcard(wildcard_model=model, litellm_params=None))
|
||||
|
||||
for model in frozenset(wildcard_routes):
|
||||
unique_models.remove(model)
|
||||
|
||||
return all_wildcard_models
|
||||
def _wildcard_route_models(
|
||||
wildcard_route: str,
|
||||
llm_router: Router | None,
|
||||
team_id: str | None,
|
||||
routes_to_model: Callable[[str, str], bool],
|
||||
) -> Iterator[str]:
|
||||
router_models: Final = llm_router.get_model_list(model_name=wildcard_route, team_id=team_id) if llm_router else None
|
||||
if not router_models:
|
||||
yield from get_known_models_from_wildcard(wildcard_model=wildcard_route, litellm_params=None)
|
||||
return
|
||||
for router_model in router_models:
|
||||
yield from get_known_models_from_wildcard(
|
||||
wildcard_model=wildcard_route,
|
||||
litellm_params=LiteLLM_Params(**router_model["litellm_params"]),
|
||||
routes_to_model=routes_to_model,
|
||||
)
|
||||
|
||||
|
||||
def get_all_fallbacks(
|
||||
|
|
|
|||
|
|
@ -559,7 +559,7 @@ def get_team_provider_credentials(
|
|||
return None
|
||||
|
||||
from litellm.proxy._types import SpecialModelNames
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list, get_key_models
|
||||
from litellm.proxy.auth.model_checks import get_key_models, iter_complete_model_list
|
||||
|
||||
team_id: Final = user_api_key_dict.team_id
|
||||
team_models: Final = user_api_key_dict.team_models or []
|
||||
|
|
@ -637,35 +637,29 @@ def get_team_provider_credentials(
|
|||
# effective allowlist (sentinels and access groups already expanded by
|
||||
# get_key_models) wins when set; otherwise the team's allowlist applies.
|
||||
# The all-proxy-models sentinel isn't expanded by
|
||||
# get_complete_model_list, so normalize it to an empty allowlist, which
|
||||
# iter_complete_model_list, so normalize it to an empty allowlist, which
|
||||
# defers to the team-scoped proxy model list. A team or key with a
|
||||
# restricted allowlist (e.g. anthropic only) therefore never resolves
|
||||
# another provider's key.
|
||||
grants_all_models: Final = SpecialModelNames.all_proxy_models.value in team_models
|
||||
effective_team_models: Final = [] if grants_all_models else team_models
|
||||
|
||||
models_to_try: Final = list(
|
||||
dict.fromkeys(
|
||||
get_complete_model_list(
|
||||
key_models=list(key_model_allowlist),
|
||||
team_models=effective_team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
return_wildcard_routes=True,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=True,
|
||||
team_id=team_id,
|
||||
)
|
||||
)
|
||||
models_to_try: Final = iter_complete_model_list(
|
||||
key_models=key_model_allowlist,
|
||||
team_models=effective_team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
return_wildcard_routes=True,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=True,
|
||||
team_id=team_id,
|
||||
)
|
||||
return next(
|
||||
(credentials for credentials in map(_provider_credentials, models_to_try) if credentials is not None),
|
||||
None,
|
||||
)
|
||||
for model_name in models_to_try:
|
||||
credentials = _provider_credentials(model_name)
|
||||
if credentials is not None:
|
||||
return credentials
|
||||
|
||||
return None
|
||||
|
||||
|
||||
HTTP_HEADER_VALUE: Final = re.compile(r"[\x21-\x7e]+(?:[ \t]+[\x21-\x7e]+)*")
|
||||
|
|
|
|||
|
|
@ -806,7 +806,7 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve
|
|||
]
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
custom_llm_provider="hosted_vllm",
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
)
|
||||
assert await asyncio.to_thread(listing_started.wait, 3)
|
||||
|
|
@ -814,7 +814,46 @@ async def test_a_hung_wildcard_listing_during_a_credential_lookup_leaves_the_eve
|
|||
await lookup
|
||||
|
||||
assert listing_answered.is_set()
|
||||
assert data.get("api_base") == "http://vllm-credential-lookup.test:8000/v1"
|
||||
assert data == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"custom_llm_provider, vllm_api_base, applied_credential",
|
||||
[
|
||||
("openai", "http://vllm-beside-openai.test:8000/v1", ("api_key", "openai-key")),
|
||||
("hosted_vllm", "http://vllm-route-match.test:8000/v1", ("api_base", "http://vllm-route-match.test:8000/v1")),
|
||||
],
|
||||
)
|
||||
async def test_a_credential_lookup_a_configured_model_answers_never_lists_the_wildcard_server(
|
||||
monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, vllm_api_base: str, applied_credential: tuple[str, str]
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
monkeypatch.setattr(litellm, "check_provider_endpoint", True)
|
||||
data: Final[dict[str, object]] = {}
|
||||
|
||||
with respx.mock(assert_all_called=False) as vllm_server:
|
||||
listing: Final = vllm_server.get(f"{vllm_api_base}/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "qwen2.5"}]})
|
||||
)
|
||||
await apply_team_provider_credentials(
|
||||
data=data,
|
||||
llm_router=Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "hosted_vllm/*",
|
||||
"litellm_params": {"model": "hosted_vllm/*", "api_base": vllm_api_base},
|
||||
},
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "openai-key"}},
|
||||
]
|
||||
),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
assert not listing.called
|
||||
assert data.get(applied_credential[0]) == applied_credential[1]
|
||||
|
||||
|
||||
def _apply_team_a_openai_credentials(data: dict[str, object]) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue