fix(proxy): stop GET /v1/models from pegging CPU on wildcard routes

litellm/proxy/utils.py

@@ -6124,7 +6124,7 @@ def create_model_info_response(
     # to the base response rather than 500-ing the whole listing.
     if llm_router is not None:
         try:
-            model_group_info = llm_router.get_model_group_info(model_id)
+            model_group_info = llm_router._cached_get_model_group_info(model_id)
         except Exception as e:
             verbose_proxy_logger.debug(
                 "create_model_info_response: get_model_group_info failed for %s: %s",

litellm/router.py

@@ -8745,11 +8745,12 @@ class Router:
         model_list = self.get_model_list(model_name=model_group)
         if model_list is None:
             return None
+        is_wildcard_group = self.pattern_router.is_match(model_group)
         for model in model_list:
             is_match = False
             if "model_name" in model and model["model_name"] == model_group:  # exact match
                 is_match = True
-            elif "model_name" in model and self.pattern_router.route(model_group) is not None:  # wildcard model
+            elif "model_name" in model and is_wildcard_group:  # wildcard model
                 is_match = True
 
             if not is_match:

litellm/router_utils/pattern_match_deployments.py

@@ -109,6 +109,42 @@ class PatternMatchRouter:
 
         return new_deployments
 
+    def _find_matching_pattern(
+        self, request: str | None, filtered_model_names: list[str] | None = None
+    ) -> tuple[Match, list[dict]] | None:
+        """
+        Find the most specific pattern matching the request without copying any deployment.
+
+        Returns the regex match and the raw (uncopied) deployments for that pattern, or None.
+        """
+        if request is None:
+            return None
+
+        sorted_patterns = PatternUtils.sorted_patterns(self.patterns)
+        regex_filtered_model_names = (
+            [self._pattern_to_regex(m) for m in filtered_model_names] if filtered_model_names is not None else []
+        )
+        for pattern, llm_deployments in sorted_patterns:
+            if filtered_model_names is not None and pattern not in regex_filtered_model_names:
+                continue
+            pattern_match = re.match(pattern, request)
+            if pattern_match:
+                return pattern_match, llm_deployments
+        return None
+
+    def is_match(self, request: str | None, filtered_model_names: list[str] | None = None) -> bool:
+        """
+        Return whether request matches a wildcard pattern, without deep-copying deployments.
+
+        Use this instead of ``route(...) is not None`` when only the boolean result is needed;
+        ``route`` deep-copies every matched deployment, which is expensive on hot paths.
+        """
+        try:
+            return self._find_matching_pattern(request, filtered_model_names) is not None
+        except Exception as e:
+            verbose_router_logger.debug(f"Error in PatternMatchRouter.is_match: {str(e)}")
+            return False
+
     def route(self, request: Optional[str], filtered_model_names: Optional[List[str]] = None) -> Optional[List[Dict]]:
         """
         Route a requested model to the corresponding llm deployments based on the regex pattern
@@ -124,21 +160,11 @@ class PatternMatchRouter:
             Optional[List[Deployment]]: llm deployments
         """
         try:
-            if request is None:
+            matched = self._find_matching_pattern(request, filtered_model_names)
+            if matched is None:
                 return None
-
-            sorted_patterns = PatternUtils.sorted_patterns(self.patterns)
-            regex_filtered_model_names = (
-                [self._pattern_to_regex(m) for m in filtered_model_names] if filtered_model_names is not None else []
-            )
-            for pattern, llm_deployments in sorted_patterns:
-                if filtered_model_names is not None and pattern not in regex_filtered_model_names:
-                    continue
-                pattern_match = re.match(pattern, request)
-                if pattern_match:
-                    return self._return_pattern_matched_deployments(
-                        matched_pattern=pattern_match, deployments=llm_deployments
-                    )
+            pattern_match, llm_deployments = matched
+            return self._return_pattern_matched_deployments(matched_pattern=pattern_match, deployments=llm_deployments)
         except Exception as e:
             verbose_router_logger.debug(f"Error in PatternMatchRouter.route: {str(e)}")
 

tests/test_litellm/proxy/test_proxy_utils.py

@@ -482,7 +482,7 @@ from litellm.types.router import ModelGroupInfo
 
 def _router_returning(model_group_info):
     router = MagicMock()
-    router.get_model_group_info = MagicMock(return_value=model_group_info)
+    router._cached_get_model_group_info = MagicMock(return_value=model_group_info)
     return router
 
 
@@ -500,7 +500,7 @@ def test_create_model_info_response_includes_max_tokens_when_available():
         model_id="qwen-vllm", provider="openai", llm_router=router
     )
 
-    router.get_model_group_info.assert_called_once_with("qwen-vllm")
+    router._cached_get_model_group_info.assert_called_once_with("qwen-vllm")
     assert response["id"] == "qwen-vllm"
     assert response["object"] == "model"
     assert response["max_input_tokens"] == 32768
@@ -582,7 +582,7 @@ def test_create_model_info_response_degrades_when_group_info_raises():
     # A malformed deployment must not turn the listing into a 500; the entry
     # falls back to the base fields without limits.
     router = MagicMock()
-    router.get_model_group_info = MagicMock(side_effect=ValueError("bad deployment"))
+    router._cached_get_model_group_info = MagicMock(side_effect=ValueError("bad deployment"))
 
     response = create_model_info_response(
         model_id="broken", provider="openai", llm_router=router

tests/test_litellm/router_utils/test_pattern_match_deployments.py

@@ -0,0 +1,50 @@
+import copy as copy_module
+from unittest.mock import patch
+
+from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
+
+
+def _openai_wildcard_router() -> PatternMatchRouter:
+    router = PatternMatchRouter()
+    router.add_pattern(
+        "openai/*",
+        {"model_name": "openai/*", "litellm_params": {"model": "openai/*"}, "model_info": {}},
+    )
+    return router
+
+
+def test_is_match_reports_membership_without_deepcopy():
+    """
+    Regression for #33636: is_match must report pattern membership without the
+    per-deployment copy.deepcopy that route performs. route deep-copied the full
+    deployment (including model_info) for every matched deployment, and it was
+    called once per model on the GET /v1/models hot path.
+    """
+    router = _openai_wildcard_router()
+
+    with patch.object(copy_module, "deepcopy", wraps=copy_module.deepcopy) as spy_deepcopy:
+        assert router.is_match("openai/gpt-4o-mini") is True
+        assert router.is_match("anthropic/claude-3") is False
+
+    assert spy_deepcopy.call_count == 0
+
+
+def test_is_match_handles_none_request():
+    assert _openai_wildcard_router().is_match(None) is False
+
+
+def test_is_match_matches_route_truthiness():
+    router = _openai_wildcard_router()
+    for request in ("openai/gpt-4o-mini", "anthropic/claude-3", "openai/"):
+        assert router.is_match(request) is (router.route(request) is not None)
+
+
+def test_route_still_copies_matched_deployments():
+    router = _openai_wildcard_router()
+
+    with patch.object(copy_module, "deepcopy", wraps=copy_module.deepcopy) as spy_deepcopy:
+        deployments = router.route("openai/gpt-4o-mini")
+
+    assert deployments is not None
+    assert deployments[0]["litellm_params"]["model"] == "openai/gpt-4o-mini"
+    assert spy_deepcopy.call_count >= 1

tests/test_litellm/test_router.py

@@ -82,6 +82,42 @@ def test_router_with_model_info_and_model_group():
     )
 
 
+def test_set_model_group_info_wildcard_avoids_per_deployment_deepcopy():
+    """
+    Regression for #33636: _set_model_group_info must not call pattern_router.route
+    (which deep-copies every matched deployment) once per deployment in the group
+    loop. That made GET /v1/models O(n^2) in deepcopies and pegged CPU for minutes
+    on wildcard routes. The loop now uses the deepcopy-free is_match check instead.
+    """
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "openai/*",
+                "litellm_params": {"model": "openai/*", "api_key": "sk-xxx"},
+            },
+            {
+                "model_name": "openai/*",
+                "litellm_params": {"model": "openai/*", "api_key": "sk-yyy"},
+            },
+        ],
+    )
+
+    with patch.object(
+        router.pattern_router,
+        "route",
+        wraps=router.pattern_router.route,
+    ) as spy_route, patch.object(
+        router.pattern_router,
+        "is_match",
+        wraps=router.pattern_router.is_match,
+    ) as spy_is_match:
+        info = router.get_model_group_info("openai/gpt-4o-mini")
+
+    assert info is not None
+    assert spy_route.call_count <= 1
+    assert spy_is_match.call_count >= 1
+
+
 def test_router_model_group_encrypted_content_affinity_callback_registration():
     from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
         DeploymentAffinityCheck,