From cd7933ecefbb84d3e954ea81bdca7c0d1093cae5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 17 Jul 2026 04:14:43 +0000 Subject: [PATCH] fix(proxy): stop GET /v1/models from pegging CPU on large deployments /v1/models built a ModelInfo per listed model and each entry recomputed group metadata via the uncached Router.get_model_group_info, expanded wildcard routes with a full copy.deepcopy per matched deployment, and re-ran pattern_router.route(model_group) inside the per-deployment loop. On large deployment lists this repeated deep copying blocked the event loop for minutes and failed health checks (#33636). create_model_info_response now reads limits through the lru_cached _cached_get_model_group_info; wildcard expansion shallow-copies the deployment and its litellm_params (only litellm_params["model"] is rewritten); and _set_model_group_info drops the redundant per-deployment route re-check, since get_model_list already normalizes a matched wildcard deployment's model_name to the requested group. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/utils.py | 2 +- litellm/router.py | 8 +---- .../router_utils/pattern_match_deployments.py | 24 +++++++------ tests/test_litellm/proxy/test_proxy_utils.py | 10 ++++-- .../test_pattern_match_deployments.py | 36 +++++++++++++++++++ tests/test_litellm/test_router.py | 30 ++++++++++++++++ 6 files changed, 88 insertions(+), 22 deletions(-) create mode 100644 tests/test_litellm/router_utils/test_pattern_match_deployments.py diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9f36e729330..25f94adf8cc 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -6124,7 +6124,7 @@ def create_model_info_response( # to the base response rather than 500-ing the whole listing. if llm_router is not None: try: - model_group_info = llm_router.get_model_group_info(model_id) + model_group_info = llm_router._cached_get_model_group_info(model_id) except Exception as e: verbose_proxy_logger.debug( "create_model_info_response: get_model_group_info failed for %s: %s", diff --git a/litellm/router.py b/litellm/router.py index 78e156801f8..6f3840077ef 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8746,13 +8746,7 @@ class Router: if model_list is None: return None for model in model_list: - is_match = False - if "model_name" in model and model["model_name"] == model_group: # exact match - is_match = True - elif "model_name" in model and self.pattern_router.route(model_group) is not None: # wildcard model - is_match = True - - if not is_match: + if "model_name" not in model or model["model_name"] != model_group: continue # model in model group found # litellm_params = LiteLLM_Params(**model["litellm_params"]) # type: ignore diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index c08f8e95cf4..a5a6188003b 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -2,7 +2,6 @@ Class to handle llm wildcard routing and regex pattern matching """ -import copy import re from re import Match from typing import Dict, List, Optional, Tuple @@ -98,16 +97,19 @@ class PatternMatchRouter: return re.escape(pattern).replace(r"\*", "(.*)") def _return_pattern_matched_deployments(self, matched_pattern: Match, deployments: List[Dict]) -> List[Dict]: - new_deployments = [] - for deployment in deployments: - new_deployment = copy.deepcopy(deployment) - new_deployment["litellm_params"]["model"] = PatternMatchRouter.set_deployment_model_name( - matched_pattern=matched_pattern, - litellm_deployment_litellm_model=deployment["litellm_params"]["model"], - ) - new_deployments.append(new_deployment) - - return new_deployments + return [ + { + **deployment, + "litellm_params": { + **deployment["litellm_params"], + "model": PatternMatchRouter.set_deployment_model_name( + matched_pattern=matched_pattern, + litellm_deployment_litellm_model=deployment["litellm_params"]["model"], + ), + }, + } + for deployment in deployments + ] def route(self, request: Optional[str], filtered_model_names: Optional[List[str]] = None) -> Optional[List[Dict]]: """ diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py index a909c510581..644b434b884 100644 --- a/tests/test_litellm/proxy/test_proxy_utils.py +++ b/tests/test_litellm/proxy/test_proxy_utils.py @@ -482,7 +482,7 @@ from litellm.types.router import ModelGroupInfo def _router_returning(model_group_info): router = MagicMock() - router.get_model_group_info = MagicMock(return_value=model_group_info) + router._cached_get_model_group_info = MagicMock(return_value=model_group_info) return router @@ -500,7 +500,11 @@ def test_create_model_info_response_includes_max_tokens_when_available(): model_id="qwen-vllm", provider="openai", llm_router=router ) - router.get_model_group_info.assert_called_once_with("qwen-vllm") + # /v1/models must read limits through the lru_cached accessor, not the + # uncached public method that recomputes group info per listed model and + # pegged the event loop for minutes (issue #33636). + router._cached_get_model_group_info.assert_called_once_with("qwen-vllm") + router.get_model_group_info.assert_not_called() assert response["id"] == "qwen-vllm" assert response["object"] == "model" assert response["max_input_tokens"] == 32768 @@ -582,7 +586,7 @@ def test_create_model_info_response_degrades_when_group_info_raises(): # A malformed deployment must not turn the listing into a 500; the entry # falls back to the base fields without limits. router = MagicMock() - router.get_model_group_info = MagicMock(side_effect=ValueError("bad deployment")) + router._cached_get_model_group_info = MagicMock(side_effect=ValueError("bad deployment")) response = create_model_info_response( model_id="broken", provider="openai", llm_router=router diff --git a/tests/test_litellm/router_utils/test_pattern_match_deployments.py b/tests/test_litellm/router_utils/test_pattern_match_deployments.py new file mode 100644 index 00000000000..9ebba60e38a --- /dev/null +++ b/tests/test_litellm/router_utils/test_pattern_match_deployments.py @@ -0,0 +1,36 @@ +import re + +from litellm.router_utils.pattern_match_deployments import PatternMatchRouter + + +def test_return_pattern_matched_deployments_does_not_deepcopy_source(): + # Regression for issue #33636: expanding a wildcard route used to + # copy.deepcopy the whole deployment per matched deployment, which pegged + # the proxy CPU for large deployments listed via GET /v1/models. Only + # litellm_params["model"] is rewritten, so the expansion must shallow-copy: + # the source deployment stays untouched and every nested object is shared + # by identity (a reintroduced deepcopy would break the identity assert). + shared_metadata = {"tier": "gold"} + deployment = { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "sk-test", "metadata": shared_metadata}, + "model_info": {"id": "abc"}, + } + + router = PatternMatchRouter() + router.add_pattern("openai/*", deployment) + + matched = router.route("openai/gpt-4") + + assert matched is not None + (expanded,) = matched + assert expanded["litellm_params"]["model"] == "openai/gpt-4" + + # source is left untouched + assert deployment["litellm_params"]["model"] == "openai/*" + + # shallow copy: new top-level and litellm_params dicts, shared nested objects + assert expanded is not deployment + assert expanded["litellm_params"] is not deployment["litellm_params"] + assert expanded["litellm_params"]["metadata"] is shared_metadata + assert expanded["model_info"] is deployment["model_info"] diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index c2c98c8869c..98ecfb2dfb0 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1581,6 +1581,36 @@ def test_model_group_info_db_fallback_with_stringified_cost_values(): assert isinstance(result.output_cost_per_token, float) +def test_set_model_group_info_resolves_wildcard_concrete_model(): + """ + Regression for issue #33636: _set_model_group_info dropped the per-deployment + pattern_router.route(model_group) re-check, one of the repeated wildcard + expansions that pegged the proxy CPU on GET /v1/models. The loop now matches + only on model_name equality, which is safe because get_model_list already + normalizes a matched wildcard deployment's model_name to the requested group. + A concrete model served by a wildcard route must still resolve to a valid + ModelGroupInfo; if that normalization ever regresses, this fails instead of + silently dropping the group's limits. + """ + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "fake"}, + }, + ] + ) + + result = router._set_model_group_info( + model_group="openai/gpt-4o", + user_facing_model_group_name="openai/gpt-4o", + ) + + assert result is not None + assert result.model_group == "openai/gpt-4o" + assert "openai" in result.providers + + def test_get_model_access_groups_caching(): """ Test that get_model_access_groups caches the no-args result