scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-11 08:38:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 359

fix(proxy): list slash-less wildcard groups like claude-* on /v1/models and /model/info (#49832)

Browse source 
* fix(proxy): list slash-less wildcard groups like claude-* on /v1/models and /model/info

* fix(proxy): expand team-owned wildcard rows on /model/info through their public name

* fix(proxy): keep listing wildcard names wider than their deployment and skip the Azure placeholder

A group like *gemini-3* over gemini/gemini-3* lists the catalog ids the name matches and routes back, as main did on /model/info, now on /v1/models too. Wildcard listings no longer list the Azure-LLM placeholder. Team-owned wildcard rows on /model/info go back to main's behavior.

* fix(proxy): keep Azure listings on slashed wildcards and list bare deployments through custom_llm_provider

get_provider_models returns main's list again, and only slash-less names skip the Azure-LLM catalog placeholder. A slash-less name over a bare wildcard deployment now lists the catalog of its custom_llm_provider.

* test(proxy): pin custom_llm_provider guards and slashed wider-name listing

* fix(proxy): list bare ids for wider names over a custom_llm_provider deployment

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
tessellark 2026-11-03 08:14:04 -08:00 • committed by GitHub
parent baddc67041
commit 1cc7da6274
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 227 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
litellm/proxy/auth/model_checks.py
View file

@ -23,7 +23,12 @@ from litellm.router import Router
from litellm.router_api import fallback_chain
from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order
from litellm.utils import (
AZURE_MODEL_PLACEHOLDER,
ProviderConfigManager,
get_valid_models,
potential_model_names_in_lookup_order,
)
LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
_PROVIDERS_SENDING_THE_DEPLOYMENT_PATH_AS_THE_MODEL_ID: Final = frozenset({LlmProviders.TOGETHER_AI.value})
@ -375,6 +380,10 @@ def _ids_routing_to_catalog_models(
catalog_ids: Sequence[str],
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
if name_pattern.count("*") > deployment_pattern.count("*"):
return _catalog_ids_a_wider_name_matches(
name_pattern, deployment_pattern, provider, catalog_ids, routes_to_model
)
if name_pattern.count("*") != deployment_pattern.count("*") or _lists_name_fragments(
name_pattern, deployment_pattern
):
@ -419,6 +428,23 @@ def _ids_routing_to_catalog_models(
)
def _catalog_ids_a_wider_name_matches(
name_pattern: str,
deployment_pattern: str,
provider: str,
catalog_ids: Sequence[str],
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
name_regex: Final = _wildcard_name_regex(name_pattern)
return tuple(
catalog_id
for catalog_id in _ids_routing_to_catalog_models(
deployment_pattern, deployment_pattern, provider, catalog_ids, routes_to_model
)
if name_regex.fullmatch(catalog_id)
)
def _model_info_field(deployment: Mapping[str, object], field: str) -> object:
model_info: Final = deployment.get("model_info")
return model_info.get(field) if is_str_keyed_mapping(model_info) else None
@ -456,11 +482,12 @@ def _routes_to_model(
return lambda listed_id, model: routed_models is None or model in routed_models(listed_id, team_id)
def _deployment_pattern_naming_listed_ids(wildcard_suffix: str, litellm_params: LiteLLM_Params | None) -> str | None:
def _deployment_pattern_naming_listed_ids(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None:
if litellm_params is None or "/" not in litellm_params.model or not _check_wildcard_routing(litellm_params.model):
return None
name_suffix: Final = wildcard_model.split("/", 1)[-1]
deployment_suffix: Final = litellm_params.model.partition("/")[2]
return litellm_params.model if wildcard_suffix != "*" or deployment_suffix != "*" else None
return litellm_params.model if name_suffix != "*" or deployment_suffix != "*" else None
def get_known_models_from_wildcard(
@ -468,6 +495,43 @@ def get_known_models_from_wildcard(
litellm_params: LiteLLM_Params | None = None,
routes_to_model: Callable[[str, str], bool] = lambda _listed_id, _model: True,
) -> list[str]:
provider_id_prefix: Final = _custom_llm_provider_prefix(wildcard_model, litellm_params)
if litellm_params is None or provider_id_prefix is None:
return list(_known_models_from_wildcard(wildcard_model, litellm_params, routes_to_model))
return [
listed_id.removeprefix(provider_id_prefix)
for listed_id in _known_models_from_wildcard(
wildcard_model,
litellm_params.model_copy(update={"model": f"{provider_id_prefix}{litellm_params.model}"}),
lambda listed_id, model: routes_to_model(
listed_id.removeprefix(provider_id_prefix), model.removeprefix(provider_id_prefix)
),
)
]
def _custom_llm_provider_prefix(wildcard_model: str, litellm_params: LiteLLM_Params | None) -> str | None:
if (
litellm_params is None
or not litellm_params.custom_llm_provider
or "/" in wildcard_model
or wildcard_model == "*"
or "/" in litellm_params.model
or not _check_wildcard_routing(litellm_params.model)
):
return None
return f"{litellm_params.custom_llm_provider}/"
def _without_azure_placeholder(catalog_ids: Sequence[str]) -> tuple[str, ...]:
return tuple(catalog_id for catalog_id in catalog_ids if catalog_id != AZURE_MODEL_PLACEHOLDER)
def _known_models_from_wildcard(
wildcard_model: str,
litellm_params: LiteLLM_Params | None,
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
wildcard_model_to_expand: Final = (
litellm_params.model
if wildcard_model == "*"
@ -476,15 +540,14 @@ def get_known_models_from_wildcard(
and "/" in litellm_params.model
else wildcard_model
)
try:
wildcard_provider_prefix, wildcard_suffix = wildcard_model_to_expand.split("/", 1)
except ValueError: # safely fail
return []
deployment_pattern_naming_listed_ids: Final = (
wildcard_model_to_expand
if wildcard_model_to_expand != wildcard_model
else _deployment_pattern_naming_listed_ids(wildcard_suffix, litellm_params)
else _deployment_pattern_naming_listed_ids(wildcard_model, litellm_params)
)
wildcard_provider_prefix, has_provider_prefix, wildcard_suffix = wildcard_model_to_expand.partition("/")
if not has_provider_prefix and deployment_pattern_naming_listed_ids is None:
return ()
# Use provider from litellm_params when available, otherwise from wildcard prefix
# (e.g., "openai" from "openai/*" - needed for BYOK where wildcard isn't in router)
@ -501,16 +564,14 @@ def get_known_models_from_wildcard(
wildcard_models = get_provider_models(provider=provider, litellm_params=litellm_params)
if wildcard_models is None:
return []
return ()
if deployment_pattern_naming_listed_ids is not None:
return list(
_ids_routing_to_catalog_models(
name_pattern=wildcard_model,
deployment_pattern=deployment_pattern_naming_listed_ids,
provider=provider,
catalog_ids=wildcard_models,
routes_to_model=routes_to_model,
)
return _ids_routing_to_catalog_models(
name_pattern=wildcard_model,
deployment_pattern=deployment_pattern_naming_listed_ids,
provider=provider,
catalog_ids=wildcard_models if has_provider_prefix else _without_azure_placeholder(wildcard_models),
routes_to_model=routes_to_model,
)
if wildcard_suffix != "*":
model_prefix: Final = wildcard_suffix.replace("*", "")
@ -526,7 +587,7 @@ def get_known_models_from_wildcard(
# add model prefix to wildcard models
wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models]
return list(
return tuple(
dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models)
)
@ -564,16 +625,6 @@ def _names_no_deployment_serves_by_name(
return tuple(name for name in names if not deployed_by_name(name, team_id))
def _wildcard_pattern_to_expand(model_name: str, litellm_model: str) -> str | None:
if not _check_wildcard_routing(model_name):
return None
if "/" in model_name or model_name == "*":
return model_name
if _check_wildcard_routing(litellm_model):
return litellm_model
return model_name
def _can_expand_to_an_allowed_name(wildcard_pattern: str, allowed_model_names: Collection[str] | None) -> bool:
if allowed_model_names is None:
return True
@ -601,10 +652,10 @@ def expand_wildcard_deployments_for_model_info(
raw_params = deployment.get("litellm_params")
litellm_params_dict: dict[str, object] = raw_params if isinstance(raw_params, dict) else {}
litellm_model = str(litellm_params_dict.get("model") or "")
if (wildcard_pattern := _wildcard_pattern_to_expand(model_name, litellm_model)) is None:
if not _check_wildcard_routing(model_name):
expanded.append(deployment)
continue
if not _can_expand_to_an_allowed_name(wildcard_pattern, allowed_model_names):
if not _can_expand_to_an_allowed_name(model_name, allowed_model_names):
continue
try:
@ -613,7 +664,7 @@ def expand_wildcard_deployments_for_model_info(
expanded.append(deployment)
continue
expanded_names = get_known_models_from_wildcard(
wildcard_model=wildcard_pattern,
wildcard_model=model_name,
litellm_params=litellm_params,
routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)),
)

5
litellm/utils.py
View file

@ -9069,6 +9069,9 @@ def bundled_provider_models(provider: str) -> tuple[str, ...]:
return tuple(model for model in bundled if not is_hidden_as_retired(cost_map.get(model, {})))
AZURE_MODEL_PLACEHOLDER: Final = "Azure-LLM"
def get_valid_models(
check_provider_endpoint: bool | None = None,
custom_llm_provider: str | None = None,
@ -9124,7 +9127,7 @@ def get_valid_models(
continue
if provider == "azure":
valid_models.append("Azure-LLM")
valid_models.append(AZURE_MODEL_PLACEHOLDER)
elif provider_config is not None and check_provider_endpoint and provider is not None:
valid_models.extend(
_get_valid_models_from_provider_api(

145
tests/unit/proxy/auth/test_model_checks.py
View file

@ -19,6 +19,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
from litellm import Router
from litellm.utils import AZURE_MODEL_PLACEHOLDER
def test_get_team_models_for_all_models_and_team_only_models():
@ -1105,13 +1106,13 @@ def test_expand_wildcard_invalid_litellm_params_passthrough():
@pytest.mark.parametrize(
("allowed_model_names", "expected_names", "expected_listings"),
[
pytest.param(None, ["hosted_vllm/qwen2.5"], 1, id="unscoped"),
pytest.param(["hosted_vllm/qwen2.5"], ["hosted_vllm/qwen2.5"], 1, id="granted-a-listed-id"),
pytest.param(["qwen-open"], [], 0, id="granted-a-name-the-alias-matches"),
pytest.param(None, ["qwen-qwen2.5"], 1, id="unscoped"),
pytest.param(["qwen-qwen2.5"], ["qwen-qwen2.5"], 1, id="granted-a-listed-name"),
pytest.param(["hosted_vllm/qwen2.5"], [], 0, id="granted-only-the-provider-id"),
],
)
@respx.mock
def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_provider_pattern(
def test_expand_wildcard_alias_without_a_slash_lists_and_filters_by_its_own_pattern(
monkeypatch: pytest.MonkeyPatch,
request: pytest.FixtureRequest,
allowed_model_names: list[str] | None,
@ -2099,6 +2100,34 @@ _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID}
pytest.param(
"*", "openai/*", "openai/gpt-partial-route-9*", set(), id="provider-prefixed-key-filter-over-catch-all"
),
pytest.param(
"gpt-partial-route-*",
"openai/gpt-partial-route-*",
None,
{f"openai/{_ROUTE_OPENAI_ID}"},
id="slashless-name-repeating-the-deployment-partial-word",
),
pytest.param("team-*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="slashless-alias-over-partial-deployment"),
pytest.param(
"team-*", "openai/*", None, {f"openai/{_ROUTE_OPENAI_ID}"}, id="slashless-alias-over-provider-catch-all"
),
pytest.param(
"team-*", "cloudflare/@cf/*/*", None, set(), id="slashless-alias-with-fewer-wildcards-than-its-deployment"
),
pytest.param(
"*partial-route-9*",
"openai/gpt-partial-route-9*",
None,
{f"openai/{_ROUTE_OPENAI_ID}"},
id="slashless-name-with-more-wildcards-than-its-deployment",
),
pytest.param(
"gemini/*partial-route-9*",
"gemini/gemini-*",
None,
{_ROUTE_GEMINI_ID},
id="slashed-name-with-more-wildcards-than-its-deployment",
),
pytest.param("*", "cloudflare/@cf/*", None, _ROUTE_CF_IDS, id="catch-all-over-partial-deployment"),
pytest.param("*", "gemini/gemini-partial-route-9*", None, set(), id="catch-all-over-mid-word-deployment"),
pytest.param("*", "openai/responses/gpt-partial-route-9*", None, set(), id="catch-all-over-mid-word-route"),
@ -2176,6 +2205,114 @@ def test_every_listed_partial_wildcard_id_routes_to_the_catalog_model_it_names(
assert listed_model_info_names == sorted(reachable)
@pytest.mark.parametrize(
("model_name", "expected_v1_models"),
[
pytest.param("azure/*", [f"azure/{AZURE_MODEL_PLACEHOLDER}"], id="provider-wildcard-lists-as-before"),
pytest.param("*", [AZURE_MODEL_PLACEHOLDER], id="catch-all-lists-as-before"),
pytest.param("my-*", [], id="slashless-alias-skips-the-placeholder"),
],
)
def test_only_slashless_azure_wildcards_skip_the_catalog_placeholder(
model_name: str, expected_v1_models: list[str]
) -> None:
from litellm import Router
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list
router: Final = Router(model_list=[{"model_name": model_name, "litellm_params": {"model": "azure/*"}}])
v1_models: Final = get_complete_model_list(
key_models=[],
team_models=[],
proxy_model_list=[model_name],
user_model=None,
infer_model_from_keys=False,
llm_router=router,
)
model_info_names: Final = [
str(row["model_name"])
for row in expand_wildcard_deployments_for_model_info(
[dict(deployment) for deployment in router.get_model_list() or []], router
)
]
assert v1_models == expected_v1_models
assert model_info_names == (expected_v1_models or [model_name])
@pytest.mark.parametrize(
("model_name", "deployment_model"),
[
pytest.param("gpt-*", "gpt-*", id="same-wildcards-as-its-deployment"),
pytest.param("*gpt-5*", "gpt-5*", id="more-wildcards-than-its-deployment"),
],
)
def test_slashless_wildcard_over_a_bare_deployment_lists_its_custom_llm_provider_catalog(
model_name: str, deployment_model: str
) -> None:
from fnmatch import fnmatchcase
from litellm import Router
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list
from litellm.utils import get_valid_models
router: Final = Router(
model_list=[
{
"model_name": model_name,
"litellm_params": {"model": deployment_model, "custom_llm_provider": "openai", "api_key": "sk-test"},
}
]
)
v1_models: Final = get_complete_model_list(
key_models=[],
team_models=[],
proxy_model_list=[model_name],
user_model=None,
infer_model_from_keys=False,
llm_router=router,
)
model_info_rows: Final = expand_wildcard_deployments_for_model_info(
[dict(deployment) for deployment in router.get_model_list() or []], router
)
openai_ids: Final = frozenset(model.removeprefix("openai/") for model in get_valid_models(custom_llm_provider="openai"))
routed_models: Final = {
listed_id: [deployment["litellm_params"]["model"] for deployment in router.wildcard_route_deployments(listed_id, None)]
for listed_id in v1_models
}
assert v1_models
assert [
listed_id for listed_id in v1_models if listed_id not in openai_ids or not fnmatchcase(listed_id, model_name)
] == []
assert {listed_id: models for listed_id, models in routed_models.items() if models != [listed_id]} == {}
assert sorted(str(row["model_name"]) for row in model_info_rows) == sorted(v1_models)
assert [row for row in model_info_rows if row["litellm_params"]["model"] != row["model_name"]] == []
@pytest.mark.parametrize(
("model_name", "deployment_model"),
[
pytest.param("*", "*", id="catch-all-over-a-bare-catch-all"),
pytest.param("foo/gpt-*", "gpt-*", id="slashed-name-over-a-bare-deployment"),
pytest.param("gpt-*", "openai/gpt-*", id="slashless-name-over-a-provider-prefixed-deployment"),
],
)
def test_custom_llm_provider_changes_no_listing_outside_slashless_names_over_bare_deployments(
model_name: str, deployment_model: str
) -> None:
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params
with_provider: Final = get_known_models_from_wildcard(
model_name, LiteLLM_Params(model=deployment_model, custom_llm_provider="openai")
)
without_provider: Final = get_known_models_from_wildcard(model_name, LiteLLM_Params(model=deployment_model))
assert with_provider == without_provider
def test_inference_provider_route_lists_every_id_the_plain_provider_wildcard_lists() -> None:
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params