From f699ab6660c358bc403893dde9aa8fba13e6f428 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 03:14:07 +0000 Subject: [PATCH] fix(proxy): stop GET /v1/models from pegging CPU on wildcard routes --- litellm/proxy/utils.py | 2 +- litellm/router.py | 3 +- .../router_utils/pattern_match_deployments.py | 54 ++++++++++++++----- tests/test_litellm/proxy/test_proxy_utils.py | 6 +-- .../test_pattern_match_deployments.py | 50 +++++++++++++++++ tests/test_litellm/test_router.py | 36 +++++++++++++ 6 files changed, 132 insertions(+), 19 deletions(-) create mode 100644 tests/test_litellm/router_utils/test_pattern_match_deployments.py diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9f36e729330..25f94adf8cc 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -6124,7 +6124,7 @@ def create_model_info_response( # to the base response rather than 500-ing the whole listing. if llm_router is not None: try: - model_group_info = llm_router.get_model_group_info(model_id) + model_group_info = llm_router._cached_get_model_group_info(model_id) except Exception as e: verbose_proxy_logger.debug( "create_model_info_response: get_model_group_info failed for %s: %s", diff --git a/litellm/router.py b/litellm/router.py index 78e156801f8..e8b9df21df7 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8745,11 +8745,12 @@ class Router: model_list = self.get_model_list(model_name=model_group) if model_list is None: return None + is_wildcard_group = self.pattern_router.is_match(model_group) for model in model_list: is_match = False if "model_name" in model and model["model_name"] == model_group: # exact match is_match = True - elif "model_name" in model and self.pattern_router.route(model_group) is not None: # wildcard model + elif "model_name" in model and is_wildcard_group: # wildcard model is_match = True if not is_match: diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index c08f8e95cf4..24587fa2448 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -109,6 +109,42 @@ class PatternMatchRouter: return new_deployments + def _find_matching_pattern( + self, request: str | None, filtered_model_names: list[str] | None = None + ) -> tuple[Match, list[dict]] | None: + """ + Find the most specific pattern matching the request without copying any deployment. + + Returns the regex match and the raw (uncopied) deployments for that pattern, or None. + """ + if request is None: + return None + + sorted_patterns = PatternUtils.sorted_patterns(self.patterns) + regex_filtered_model_names = ( + [self._pattern_to_regex(m) for m in filtered_model_names] if filtered_model_names is not None else [] + ) + for pattern, llm_deployments in sorted_patterns: + if filtered_model_names is not None and pattern not in regex_filtered_model_names: + continue + pattern_match = re.match(pattern, request) + if pattern_match: + return pattern_match, llm_deployments + return None + + def is_match(self, request: str | None, filtered_model_names: list[str] | None = None) -> bool: + """ + Return whether request matches a wildcard pattern, without deep-copying deployments. + + Use this instead of ``route(...) is not None`` when only the boolean result is needed; + ``route`` deep-copies every matched deployment, which is expensive on hot paths. + """ + try: + return self._find_matching_pattern(request, filtered_model_names) is not None + except Exception as e: + verbose_router_logger.debug(f"Error in PatternMatchRouter.is_match: {str(e)}") + return False + def route(self, request: Optional[str], filtered_model_names: Optional[List[str]] = None) -> Optional[List[Dict]]: """ Route a requested model to the corresponding llm deployments based on the regex pattern @@ -124,21 +160,11 @@ class PatternMatchRouter: Optional[List[Deployment]]: llm deployments """ try: - if request is None: + matched = self._find_matching_pattern(request, filtered_model_names) + if matched is None: return None - - sorted_patterns = PatternUtils.sorted_patterns(self.patterns) - regex_filtered_model_names = ( - [self._pattern_to_regex(m) for m in filtered_model_names] if filtered_model_names is not None else [] - ) - for pattern, llm_deployments in sorted_patterns: - if filtered_model_names is not None and pattern not in regex_filtered_model_names: - continue - pattern_match = re.match(pattern, request) - if pattern_match: - return self._return_pattern_matched_deployments( - matched_pattern=pattern_match, deployments=llm_deployments - ) + pattern_match, llm_deployments = matched + return self._return_pattern_matched_deployments(matched_pattern=pattern_match, deployments=llm_deployments) except Exception as e: verbose_router_logger.debug(f"Error in PatternMatchRouter.route: {str(e)}") diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index a909c510581..8b1defc5812 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -482,7 +482,7 @@ from litellm.types.router import ModelGroupInfo def _router_returning(model_group_info): router = MagicMock() - router.get_model_group_info = MagicMock(return_value=model_group_info) + router._cached_get_model_group_info = MagicMock(return_value=model_group_info) return router @@ -500,7 +500,7 @@ def test_create_model_info_response_includes_max_tokens_when_available(): model_id="qwen-vllm", provider="openai", llm_router=router ) - router.get_model_group_info.assert_called_once_with("qwen-vllm") + router._cached_get_model_group_info.assert_called_once_with("qwen-vllm") assert response["id"] == "qwen-vllm" assert response["object"] == "model" assert response["max_input_tokens"] == 32768 @@ -582,7 +582,7 @@ def test_create_model_info_response_degrades_when_group_info_raises(): # A malformed deployment must not turn the listing into a 500; the entry # falls back to the base fields without limits. router = MagicMock() - router.get_model_group_info = MagicMock(side_effect=ValueError("bad deployment")) + router._cached_get_model_group_info = MagicMock(side_effect=ValueError("bad deployment")) response = create_model_info_response( model_id="broken", provider="openai", llm_router=router diff --git a/tests/test_litellm/router_utils/test_pattern_match_deployments.py b/tests/test_litellm/router_utils/test_pattern_match_deployments.py new file mode 100644 index 00000000000..87a8c8c90ed --- /dev/null +++ b/tests/test_litellm/router_utils/test_pattern_match_deployments.py @@ -0,0 +1,50 @@ +import copy as copy_module +from unittest.mock import patch + +from litellm.router_utils.pattern_match_deployments import PatternMatchRouter + + +def _openai_wildcard_router() -> PatternMatchRouter: + router = PatternMatchRouter() + router.add_pattern( + "openai/*", + {"model_name": "openai/*", "litellm_params": {"model": "openai/*"}, "model_info": {}}, + ) + return router + + +def test_is_match_reports_membership_without_deepcopy(): + """ + Regression for #33636: is_match must report pattern membership without the + per-deployment copy.deepcopy that route performs. route deep-copied the full + deployment (including model_info) for every matched deployment, and it was + called once per model on the GET /v1/models hot path. + """ + router = _openai_wildcard_router() + + with patch.object(copy_module, "deepcopy", wraps=copy_module.deepcopy) as spy_deepcopy: + assert router.is_match("openai/gpt-4o-mini") is True + assert router.is_match("anthropic/claude-3") is False + + assert spy_deepcopy.call_count == 0 + + +def test_is_match_handles_none_request(): + assert _openai_wildcard_router().is_match(None) is False + + +def test_is_match_matches_route_truthiness(): + router = _openai_wildcard_router() + for request in ("openai/gpt-4o-mini", "anthropic/claude-3", "openai/"): + assert router.is_match(request) is (router.route(request) is not None) + + +def test_route_still_copies_matched_deployments(): + router = _openai_wildcard_router() + + with patch.object(copy_module, "deepcopy", wraps=copy_module.deepcopy) as spy_deepcopy: + deployments = router.route("openai/gpt-4o-mini") + + assert deployments is not None + assert deployments[0]["litellm_params"]["model"] == "openai/gpt-4o-mini" + assert spy_deepcopy.call_count >= 1 diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c2c98c8869c..a0d9773b112 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -82,6 +82,42 @@ def test_router_with_model_info_and_model_group(): ) +def test_set_model_group_info_wildcard_avoids_per_deployment_deepcopy(): + """ + Regression for #33636: _set_model_group_info must not call pattern_router.route + (which deep-copies every matched deployment) once per deployment in the group + loop. That made GET /v1/models O(n^2) in deepcopies and pegged CPU for minutes + on wildcard routes. The loop now uses the deepcopy-free is_match check instead. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "sk-xxx"}, + }, + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "sk-yyy"}, + }, + ], + ) + + with patch.object( + router.pattern_router, + "route", + wraps=router.pattern_router.route, + ) as spy_route, patch.object( + router.pattern_router, + "is_match", + wraps=router.pattern_router.is_match, + ) as spy_is_match: + info = router.get_model_group_info("openai/gpt-4o-mini") + + assert info is not None + assert spy_route.call_count <= 1 + assert spy_is_match.call_count >= 1 + + def test_router_model_group_encrypted_content_affinity_callback_registration(): from litellm.router_utils.pre_call_checks.deployment_affinity_check import ( DeploymentAffinityCheck,