scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-25 05:54:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 125

fix(proxy): list wildcard aliases that route at a word boundary and ids a sibling serves with the same model

Browse source
This commit is contained in:
mateo-berri 2026-10-19 17:57:16 -07:00
parent 73fd10413f
commit 6a5ce29e89
6 changed files with 147 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

66
litellm/proxy/auth/model_checks.py
View file

@ -1,6 +1,7 @@
# What is this?
## Common checks for /v1/models and `/model/info`
import copy
import functools
import itertools
import re
from collections.abc import Callable, Mapping, Sequence
@ -299,24 +300,40 @@ def _without_routing_segments(
def _lists_name_fragments(name_pattern: str, deployment_pattern: str) -> bool:
deployment_partial_word: Final = deployment_pattern.partition("*")[0].rpartition("/")[2]
return bool(deployment_partial_word) and not name_pattern.partition("*")[0].endswith(deployment_partial_word)
name_partial_word: Final = name_pattern.partition("*")[0].rpartition("/")[2]
return deployment_partial_word[-1:].isalnum() and not name_partial_word
def _routes_back_to(
listed_id: str, name_regex: re.Pattern[str], deployment_model_pattern: str, catalog_id: str
listed_id: str,
name_regex: re.Pattern[str],
catalog_route: tuple[str, str],
deployment_pattern: str,
routes_to_model: Callable[[str, str], bool],
) -> bool:
name_match: Final = name_regex.match(listed_id)
catalog_pattern, catalog_id = catalog_route
return (
name_match is not None
and PatternMatchRouter.set_deployment_model_name(
matched_pattern=name_match, litellm_deployment_litellm_model=deployment_model_pattern
matched_pattern=name_match, litellm_deployment_litellm_model=catalog_pattern
)
== catalog_id
and routes_to_model(
listed_id,
PatternMatchRouter.set_deployment_model_name(
matched_pattern=name_match, litellm_deployment_litellm_model=deployment_pattern
),
)
)
def _ids_routing_to_catalog_models(
name_pattern: str, deployment_pattern: str, provider: str, catalog_ids: Sequence[str]
name_pattern: str,
deployment_pattern: str,
provider: str,
catalog_ids: Sequence[str],
routes_to_model: Callable[[str, str], bool],
) -> tuple[str, ...]:
if name_pattern.count("*") != deployment_pattern.count("*") or _lists_name_fragments(
name_pattern, deployment_pattern
@ -347,7 +364,10 @@ def _ids_routing_to_catalog_models(
dict.fromkeys(
listed_id
for listed_id, catalog_id in listed_ids
if listed_id and _routes_back_to(listed_id, name_regex, deployment_model_pattern, catalog_id)
if listed_id
and _routes_back_to(
listed_id, name_regex, (deployment_model_pattern, catalog_id), deployment_pattern, routes_to_model
)
)
)
@ -362,17 +382,14 @@ def _deployment_team_id(deployment: Mapping[str, object]) -> str | None:
return team_id if isinstance(team_id, str) else None
def _routes_to_deployment(
llm_router: Router | None, deployment: Mapping[str, object], team_id: str | None
) -> Callable[[str], bool]:
deployment_id: Final = _model_info_field(deployment, "id")
return lambda listed_id: (
llm_router is None
or any(
_model_info_field(routed, "id") == deployment_id
for routed in llm_router.wildcard_route_deployments(listed_id, team_id)
)
)
def _cached_wildcard_routing(llm_router: Router | None) -> Callable[[str, str | None], frozenset[str]] | None:
return None if llm_router is None else functools.cache(llm_router.wildcard_routed_models)
def _routes_to_model(
routed_models: Callable[[str, str | None], frozenset[str]] | None, team_id: str | None
) -> Callable[[str, str], bool]:
return lambda listed_id, model: routed_models is None or model in routed_models(listed_id, team_id)
def _deployment_pattern_naming_listed_ids(wildcard_suffix: str, litellm_params: LiteLLM_Params | None) -> str | None:
@ -385,7 +402,7 @@ def _deployment_pattern_naming_listed_ids(wildcard_suffix: str, litellm_params:
def get_known_models_from_wildcard(
wildcard_model: str,
litellm_params: LiteLLM_Params | None = None,
routes_to_this_deployment: Callable[[str], bool] = lambda _listed_id: True,
routes_to_model: Callable[[str, str], bool] = lambda _listed_id, _model: True,
) -> list[str]:
wildcard_model_to_expand: Final = (
litellm_params.model
@ -418,16 +435,15 @@ def get_known_models_from_wildcard(
if wildcard_models is None:
return []
if deployment_pattern_naming_listed_ids is not None:
return [
listed_id
for listed_id in _ids_routing_to_catalog_models(
return list(
_ids_routing_to_catalog_models(
name_pattern=wildcard_model,
deployment_pattern=deployment_pattern_naming_listed_ids,
provider=provider,
catalog_ids=wildcard_models,
routes_to_model=routes_to_model,
)
if routes_to_this_deployment(listed_id)
]
)
if wildcard_suffix != "*":
model_prefix: Final = wildcard_suffix.replace("*", "")
provider_id_prefix: Final = f"{provider}/"
@ -473,6 +489,7 @@ def expand_wildcard_deployments_for_model_info(
becomes one entry per known openai model, matching /v1/models behaviour.
"""
expanded: Final[list[dict[str, object]]] = []
routed_models: Final = _cached_wildcard_routing(llm_router)
for deployment in deployments:
model_name = str(deployment.get("model_name") or "")
raw_params = deployment.get("litellm_params")
@ -500,7 +517,7 @@ def expand_wildcard_deployments_for_model_info(
expanded_names = get_known_models_from_wildcard(
wildcard_model=wildcard_pattern,
litellm_params=litellm_params,
routes_to_this_deployment=_routes_to_deployment(llm_router, deployment, _deployment_team_id(deployment)),
routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)),
)
if not expanded_names:
expanded.append(deployment)
@ -524,6 +541,7 @@ def _get_wildcard_models(
team_id: str | None = None,
) -> list[str]:
models_to_remove: Final = set()
routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id)
all_wildcard_models: Final = []
for model in unique_models:
if _check_wildcard_routing(model=model):
@ -539,7 +557,7 @@ def _get_wildcard_models(
get_known_models_from_wildcard(
wildcard_model=model,
litellm_params=LiteLLM_Params(**router_model["litellm_params"]),
routes_to_this_deployment=_routes_to_deployment(llm_router, router_model, team_id),
routes_to_model=routes_to_model,
)
)
else:

8
litellm/router.py
View file

@ -13119,6 +13119,14 @@ class Router:
return proxy_wide
return team_router.get_deployments_by_pattern(model=model)
def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]:
"""The models wildcard_route_deployments would route ``model`` to, without copying the deployments."""
proxy_wide: Final = self.pattern_router.get_models_by_pattern(model=model)
team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None
if proxy_wide or team_router is None:
return proxy_wide
return team_router.get_models_by_pattern(model=model)
def team_ptu_ceiling(self, team_id: str, model_group: str) -> PTUTeamCeiling | None:
"""The per-minute ceiling ``team_id``'s PTU shares set on the group serving ``model_group``,
else None, kept until the model list next changes."""

23
litellm/router_utils/pattern_match_deployments.py
View file

@ -3,6 +3,7 @@ Class to handle llm wildcard routing and regex pattern matching
"""
import copy
import itertools
import re
from re import Match
from typing import Final
@ -259,6 +260,28 @@ class PatternMatchRouter:
return pattern_match
return []
def get_models_by_pattern(self, model: str, custom_llm_provider: str | None = None) -> frozenset[str]:
"""The models get_deployments_by_pattern would route ``model`` to, without copying the deployments."""
provider: Final = custom_llm_provider or inferred_provider(model)
requests_by_pattern: Final = itertools.product((model, f"{provider}/{model}"), self.patterns)
matched: Final = next(
(
(pattern_match, pattern)
for request, pattern in requests_by_pattern
if (pattern_match := re.match(pattern, request)) is not None
),
None,
)
if matched is None:
return frozenset()
pattern_match, pattern = matched
return frozenset(
PatternMatchRouter.set_deployment_model_name(
matched_pattern=pattern_match, litellm_deployment_litellm_model=deployment["litellm_params"]["model"]
)
for deployment in self.patterns[pattern]
)
# Example usage:
# router = PatternRouter()

47
tests/unit/proxy/auth/test_model_checks.py
View file

@ -1306,6 +1306,20 @@ _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID}
{f"openai/{_ROUTE_OPENAI_ID}"},
id="alias-repeating-the-deployment-partial-word",
),
pytest.param(
"team/gpt-partial-route-9-prod*",
"openai/gpt-partial-route-9*",
None,
{f"openai/{_ROUTE_OPENAI_ID}"},
id="alias-extending-the-deployment-partial-word",
),
pytest.param(
"partial-route-g/*",
"gemini/gemini-partial-route-*",
None,
{_ROUTE_GEMINI_ID},
id="alias-over-deployment-cut-at-a-word-boundary",
),
pytest.param(
"cloudflare/*", "cloudflare/*", "cloudflare/@cf/*", _ROUTE_CF_IDS, id="key-filter-over-provider-wildcard"
),
@ -1453,6 +1467,39 @@ def test_catch_all_lists_no_id_a_more_specific_sibling_wildcard_routes_elsewhere
assert routed_models(model_info_names) == {"unshadowed-route-org/model-d": unshadowed_id}
def test_wildcard_lists_an_id_a_more_specific_sibling_serves_with_the_same_model() -> None:
from litellm import Router
from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list
router: Final = Router(
model_list=[
{"model_name": "openai/gpt-partial-route-9*", "litellm_params": {"model": "openai/gpt-partial-route-9*"}},
{
"model_name": "openai/gpt-partial-route-9-mini*",
"litellm_params": {"model": "openai/gpt-partial-route-9-mini*"},
},
]
)
with _registered_chat_models({_ROUTE_OPENAI_ID: "openai"}):
v1_models: Final = get_complete_model_list(
key_models=["openai/gpt-partial-route-9*"],
team_models=[],
proxy_model_list=["openai/gpt-partial-route-9*", "openai/gpt-partial-route-9-mini*"],
user_model=None,
infer_model_from_keys=False,
llm_router=router,
)
model_info_names: Final = [
str(row["model_name"])
for row in expand_wildcard_deployments_for_model_info(
[dict(deployment) for deployment in router.get_model_list() or []], router
)
]
assert v1_models == [f"openai/{_ROUTE_OPENAI_ID}"]
assert "openai/gpt-partial-route-9*" not in model_info_names
def test_prefixed_alias_over_a_bare_catch_all_lists_ids_that_route_through_it(monkeypatch: pytest.MonkeyPatch) -> None:
from litellm import Router
from litellm.proxy.auth.model_checks import get_complete_model_list

24
tests/unit/router_utils/test_pattern_match_deployments.py
View file

@ -210,6 +210,30 @@ def test_route_with_none():
assert router.route(None) is None
@pytest.mark.parametrize(
("model", "custom_llm_provider"),
[
pytest.param("openai/gpt-4o", None, id="most-specific-pattern-wins"),
pytest.param("o3", "openai", id="provider-qualified-retry"),
pytest.param("openai/o3", None, id="every-deployment-on-the-pattern"),
pytest.param("mistral/large", "mistral", id="no-pattern-matches"),
],
)
def test_get_models_by_pattern_names_the_models_get_deployments_by_pattern_routes_to(
model: str, custom_llm_provider: str | None
):
router: Final = PatternMatchRouter()
router.add_pattern("openai/*", _wildcard_deployment("openai/*"))
router.add_pattern("openai/*", {"model_name": "openai/*", "litellm_params": {"model": "azure/*"}})
router.add_pattern("openai/gpt-*", {"model_name": "openai/gpt-*", "litellm_params": {"model": "azure/gpt-*-prod"}})
routed: Final = router.get_deployments_by_pattern(model, custom_llm_provider=custom_llm_provider)
assert router.get_models_by_pattern(model, custom_llm_provider=custom_llm_provider) == frozenset(
_matched_models(routed)
)
def test_route_with_multiple_matching_patterns():
"""
Tests that the router returns the first matching pattern when there are multiple matching patterns

3
tests/unit/test_router/test_router.py
View file

@ -8616,6 +8616,9 @@ def test_wildcard_route_deployments_is_the_one_pattern_routing_serves_from(
assert tuple(Deployment.model_validate(d).model_info.id for d in served) == expected_ids
assert tuple(Deployment.model_validate(d).model_info.id for d in early_deployments) == expected_ids
assert router.wildcard_routed_models(requested, team_id) == frozenset(
Deployment.model_validate(d).litellm_params.model for d in served
)
def test_get_deployment_credentials_with_provider_aws_bedrock_runtime_endpoint():