From 6a5ce29e89d95616c0cf928cd4cd30038aca1ce9 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:57:16 -0700 Subject: [PATCH] fix(proxy): list wildcard aliases that route at a word boundary and ids a sibling serves with the same model --- litellm/proxy/auth/model_checks.py | 66 ++++++++++++------- litellm/router.py | 8 +++ .../router_utils/pattern_match_deployments.py | 23 +++++++ tests/unit/proxy/auth/test_model_checks.py | 47 +++++++++++++ .../test_pattern_match_deployments.py | 24 +++++++ tests/unit/test_router/test_router.py | 3 + 6 files changed, 147 insertions(+), 24 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index a229d905ad7..c4c538083ee 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -1,6 +1,7 @@ # What is this? ## Common checks for /v1/models and `/model/info` import copy +import functools import itertools import re from collections.abc import Callable, Mapping, Sequence @@ -299,24 +300,40 @@ def _without_routing_segments( def _lists_name_fragments(name_pattern: str, deployment_pattern: str) -> bool: deployment_partial_word: Final = deployment_pattern.partition("*")[0].rpartition("/")[2] - return bool(deployment_partial_word) and not name_pattern.partition("*")[0].endswith(deployment_partial_word) + name_partial_word: Final = name_pattern.partition("*")[0].rpartition("/")[2] + return deployment_partial_word[-1:].isalnum() and not name_partial_word def _routes_back_to( - listed_id: str, name_regex: re.Pattern[str], deployment_model_pattern: str, catalog_id: str + listed_id: str, + name_regex: re.Pattern[str], + catalog_route: tuple[str, str], + deployment_pattern: str, + routes_to_model: Callable[[str, str], bool], ) -> bool: name_match: Final = name_regex.match(listed_id) + catalog_pattern, catalog_id = catalog_route return ( name_match is not None and PatternMatchRouter.set_deployment_model_name( - matched_pattern=name_match, litellm_deployment_litellm_model=deployment_model_pattern + matched_pattern=name_match, litellm_deployment_litellm_model=catalog_pattern ) == catalog_id + and routes_to_model( + listed_id, + PatternMatchRouter.set_deployment_model_name( + matched_pattern=name_match, litellm_deployment_litellm_model=deployment_pattern + ), + ) ) def _ids_routing_to_catalog_models( - name_pattern: str, deployment_pattern: str, provider: str, catalog_ids: Sequence[str] + name_pattern: str, + deployment_pattern: str, + provider: str, + catalog_ids: Sequence[str], + routes_to_model: Callable[[str, str], bool], ) -> tuple[str, ...]: if name_pattern.count("*") != deployment_pattern.count("*") or _lists_name_fragments( name_pattern, deployment_pattern @@ -347,7 +364,10 @@ def _ids_routing_to_catalog_models( dict.fromkeys( listed_id for listed_id, catalog_id in listed_ids - if listed_id and _routes_back_to(listed_id, name_regex, deployment_model_pattern, catalog_id) + if listed_id + and _routes_back_to( + listed_id, name_regex, (deployment_model_pattern, catalog_id), deployment_pattern, routes_to_model + ) ) ) @@ -362,17 +382,14 @@ def _deployment_team_id(deployment: Mapping[str, object]) -> str | None: return team_id if isinstance(team_id, str) else None -def _routes_to_deployment( - llm_router: Router | None, deployment: Mapping[str, object], team_id: str | None -) -> Callable[[str], bool]: - deployment_id: Final = _model_info_field(deployment, "id") - return lambda listed_id: ( - llm_router is None - or any( - _model_info_field(routed, "id") == deployment_id - for routed in llm_router.wildcard_route_deployments(listed_id, team_id) - ) - ) +def _cached_wildcard_routing(llm_router: Router | None) -> Callable[[str, str | None], frozenset[str]] | None: + return None if llm_router is None else functools.cache(llm_router.wildcard_routed_models) + + +def _routes_to_model( + routed_models: Callable[[str, str | None], frozenset[str]] | None, team_id: str | None +) -> Callable[[str, str], bool]: + return lambda listed_id, model: routed_models is None or model in routed_models(listed_id, team_id) def _deployment_pattern_naming_listed_ids(wildcard_suffix: str, litellm_params: LiteLLM_Params | None) -> str | None: @@ -385,7 +402,7 @@ def _deployment_pattern_naming_listed_ids(wildcard_suffix: str, litellm_params: def get_known_models_from_wildcard( wildcard_model: str, litellm_params: LiteLLM_Params | None = None, - routes_to_this_deployment: Callable[[str], bool] = lambda _listed_id: True, + routes_to_model: Callable[[str, str], bool] = lambda _listed_id, _model: True, ) -> list[str]: wildcard_model_to_expand: Final = ( litellm_params.model @@ -418,16 +435,15 @@ def get_known_models_from_wildcard( if wildcard_models is None: return [] if deployment_pattern_naming_listed_ids is not None: - return [ - listed_id - for listed_id in _ids_routing_to_catalog_models( + return list( + _ids_routing_to_catalog_models( name_pattern=wildcard_model, deployment_pattern=deployment_pattern_naming_listed_ids, provider=provider, catalog_ids=wildcard_models, + routes_to_model=routes_to_model, ) - if routes_to_this_deployment(listed_id) - ] + ) if wildcard_suffix != "*": model_prefix: Final = wildcard_suffix.replace("*", "") provider_id_prefix: Final = f"{provider}/" @@ -473,6 +489,7 @@ def expand_wildcard_deployments_for_model_info( becomes one entry per known openai model, matching /v1/models behaviour. """ expanded: Final[list[dict[str, object]]] = [] + routed_models: Final = _cached_wildcard_routing(llm_router) for deployment in deployments: model_name = str(deployment.get("model_name") or "") raw_params = deployment.get("litellm_params") @@ -500,7 +517,7 @@ def expand_wildcard_deployments_for_model_info( expanded_names = get_known_models_from_wildcard( wildcard_model=wildcard_pattern, litellm_params=litellm_params, - routes_to_this_deployment=_routes_to_deployment(llm_router, deployment, _deployment_team_id(deployment)), + routes_to_model=_routes_to_model(routed_models, _deployment_team_id(deployment)), ) if not expanded_names: expanded.append(deployment) @@ -524,6 +541,7 @@ def _get_wildcard_models( team_id: str | None = None, ) -> list[str]: models_to_remove: Final = set() + routes_to_model: Final = _routes_to_model(_cached_wildcard_routing(llm_router), team_id) all_wildcard_models: Final = [] for model in unique_models: if _check_wildcard_routing(model=model): @@ -539,7 +557,7 @@ def _get_wildcard_models( get_known_models_from_wildcard( wildcard_model=model, litellm_params=LiteLLM_Params(**router_model["litellm_params"]), - routes_to_this_deployment=_routes_to_deployment(llm_router, router_model, team_id), + routes_to_model=routes_to_model, ) ) else: diff --git a/litellm/router.py b/litellm/router.py index 8d77fd0dac5..d6834cc8ea8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -13119,6 +13119,14 @@ class Router: return proxy_wide return team_router.get_deployments_by_pattern(model=model) + def wildcard_routed_models(self, model: str, team_id: str | None) -> frozenset[str]: + """The models wildcard_route_deployments would route ``model`` to, without copying the deployments.""" + proxy_wide: Final = self.pattern_router.get_models_by_pattern(model=model) + team_router: Final = self.team_pattern_routers.get(team_id) if team_id is not None else None + if proxy_wide or team_router is None: + return proxy_wide + return team_router.get_models_by_pattern(model=model) + def team_ptu_ceiling(self, team_id: str, model_group: str) -> PTUTeamCeiling | None: """The per-minute ceiling ``team_id``'s PTU shares set on the group serving ``model_group``, else None, kept until the model list next changes.""" diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index 118106e4b43..f921f907f4c 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -3,6 +3,7 @@ Class to handle llm wildcard routing and regex pattern matching """ import copy +import itertools import re from re import Match from typing import Final @@ -259,6 +260,28 @@ class PatternMatchRouter: return pattern_match return [] + def get_models_by_pattern(self, model: str, custom_llm_provider: str | None = None) -> frozenset[str]: + """The models get_deployments_by_pattern would route ``model`` to, without copying the deployments.""" + provider: Final = custom_llm_provider or inferred_provider(model) + requests_by_pattern: Final = itertools.product((model, f"{provider}/{model}"), self.patterns) + matched: Final = next( + ( + (pattern_match, pattern) + for request, pattern in requests_by_pattern + if (pattern_match := re.match(pattern, request)) is not None + ), + None, + ) + if matched is None: + return frozenset() + pattern_match, pattern = matched + return frozenset( + PatternMatchRouter.set_deployment_model_name( + matched_pattern=pattern_match, litellm_deployment_litellm_model=deployment["litellm_params"]["model"] + ) + for deployment in self.patterns[pattern] + ) + # Example usage: # router = PatternRouter() diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 1d58734745e..83761620cc9 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -1306,6 +1306,20 @@ _ROUTE_CF_IDS: Final = {_ROUTE_CF_ID, _ROUTE_CF_SUFFIXED_ID} {f"openai/{_ROUTE_OPENAI_ID}"}, id="alias-repeating-the-deployment-partial-word", ), + pytest.param( + "team/gpt-partial-route-9-prod*", + "openai/gpt-partial-route-9*", + None, + {f"openai/{_ROUTE_OPENAI_ID}"}, + id="alias-extending-the-deployment-partial-word", + ), + pytest.param( + "partial-route-g/*", + "gemini/gemini-partial-route-*", + None, + {_ROUTE_GEMINI_ID}, + id="alias-over-deployment-cut-at-a-word-boundary", + ), pytest.param( "cloudflare/*", "cloudflare/*", "cloudflare/@cf/*", _ROUTE_CF_IDS, id="key-filter-over-provider-wildcard" ), @@ -1453,6 +1467,39 @@ def test_catch_all_lists_no_id_a_more_specific_sibling_wildcard_routes_elsewhere assert routed_models(model_info_names) == {"unshadowed-route-org/model-d": unshadowed_id} +def test_wildcard_lists_an_id_a_more_specific_sibling_serves_with_the_same_model() -> None: + from litellm import Router + from litellm.proxy.auth.model_checks import expand_wildcard_deployments_for_model_info, get_complete_model_list + + router: Final = Router( + model_list=[ + {"model_name": "openai/gpt-partial-route-9*", "litellm_params": {"model": "openai/gpt-partial-route-9*"}}, + { + "model_name": "openai/gpt-partial-route-9-mini*", + "litellm_params": {"model": "openai/gpt-partial-route-9-mini*"}, + }, + ] + ) + with _registered_chat_models({_ROUTE_OPENAI_ID: "openai"}): + v1_models: Final = get_complete_model_list( + key_models=["openai/gpt-partial-route-9*"], + team_models=[], + proxy_model_list=["openai/gpt-partial-route-9*", "openai/gpt-partial-route-9-mini*"], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + model_info_names: Final = [ + str(row["model_name"]) + for row in expand_wildcard_deployments_for_model_info( + [dict(deployment) for deployment in router.get_model_list() or []], router + ) + ] + + assert v1_models == [f"openai/{_ROUTE_OPENAI_ID}"] + assert "openai/gpt-partial-route-9*" not in model_info_names + + def test_prefixed_alias_over_a_bare_catch_all_lists_ids_that_route_through_it(monkeypatch: pytest.MonkeyPatch) -> None: from litellm import Router from litellm.proxy.auth.model_checks import get_complete_model_list diff --git a/tests/unit/router_utils/test_pattern_match_deployments.py b/tests/unit/router_utils/test_pattern_match_deployments.py index 01af021112b..0bb33064c68 100644 --- a/tests/unit/router_utils/test_pattern_match_deployments.py +++ b/tests/unit/router_utils/test_pattern_match_deployments.py @@ -210,6 +210,30 @@ def test_route_with_none(): assert router.route(None) is None +@pytest.mark.parametrize( + ("model", "custom_llm_provider"), + [ + pytest.param("openai/gpt-4o", None, id="most-specific-pattern-wins"), + pytest.param("o3", "openai", id="provider-qualified-retry"), + pytest.param("openai/o3", None, id="every-deployment-on-the-pattern"), + pytest.param("mistral/large", "mistral", id="no-pattern-matches"), + ], +) +def test_get_models_by_pattern_names_the_models_get_deployments_by_pattern_routes_to( + model: str, custom_llm_provider: str | None +): + router: Final = PatternMatchRouter() + router.add_pattern("openai/*", _wildcard_deployment("openai/*")) + router.add_pattern("openai/*", {"model_name": "openai/*", "litellm_params": {"model": "azure/*"}}) + router.add_pattern("openai/gpt-*", {"model_name": "openai/gpt-*", "litellm_params": {"model": "azure/gpt-*-prod"}}) + + routed: Final = router.get_deployments_by_pattern(model, custom_llm_provider=custom_llm_provider) + + assert router.get_models_by_pattern(model, custom_llm_provider=custom_llm_provider) == frozenset( + _matched_models(routed) + ) + + def test_route_with_multiple_matching_patterns(): """ Tests that the router returns the first matching pattern when there are multiple matching patterns diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index bd3cf738af8..d521528b334 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -8616,6 +8616,9 @@ def test_wildcard_route_deployments_is_the_one_pattern_routing_serves_from( assert tuple(Deployment.model_validate(d).model_info.id for d in served) == expected_ids assert tuple(Deployment.model_validate(d).model_info.id for d in early_deployments) == expected_ids + assert router.wildcard_routed_models(requested, team_id) == frozenset( + Deployment.model_validate(d).litellm_params.model for d in served + ) def test_get_deployment_credentials_with_provider_aws_bedrock_runtime_endpoint():