fix(pattern_matching_router.py): update model name using correct function

litellm/proxy/_new_secret_config.yaml

@@ -13,6 +13,10 @@ model_list:
   - model_name: gemini-1.5-flash-002
     litellm_params:
       model: gemini/gemini-1.5-flash-002
+  - model_name: "llmengine/*"
+    litellm_params:
+      model: "openai/*"
+      api_key: sk-1234
 
 # litellm_settings:
 #   fallbacks: [{ "claude-3-5-sonnet-20240620": ["claude-3-5-sonnet-aihubmix"] }]
@@ -20,53 +24,53 @@ model_list:
 #   default_redis_batch_cache_expiry: 10
 
 
-litellm_settings:
-  cache: True
-  cache_params:
-    type: redis
+# litellm_settings:
+#   cache: True
+#   cache_params:
+#     type: redis
 
-    # disable caching on the actual API call
-    supported_call_types: []
+#     # disable caching on the actual API call
+#     supported_call_types: []
 
-    # see https://docs.litellm.ai/docs/proxy/prod#3-use-redis-porthost-password-not-redis_url
-    host: os.environ/REDIS_HOST
-    port: os.environ/REDIS_PORT
-    password: os.environ/REDIS_PASSWORD
+#     # see https://docs.litellm.ai/docs/proxy/prod#3-use-redis-porthost-password-not-redis_url
+#     host: os.environ/REDIS_HOST
+#     port: os.environ/REDIS_PORT
+#     password: os.environ/REDIS_PASSWORD
 
   # see https://docs.litellm.ai/docs/proxy/caching#turn-on-batch_redis_requests
   # see https://docs.litellm.ai/docs/proxy/prometheus
-  callbacks: ['prometheus', 'otel']
+  # callbacks: ['prometheus', 'otel']
 
-  # # see https://docs.litellm.ai/docs/proxy/logging#logging-proxy-inputoutput---sentry
-  failure_callback: ['sentry']
-  service_callback: ['prometheus_system']
+  # # # see https://docs.litellm.ai/docs/proxy/logging#logging-proxy-inputoutput---sentry
+  # failure_callback: ['sentry']
+  # service_callback: ['prometheus_system']
 
   # redact_user_api_key_info: true
 
 
-router_settings:
-  routing_strategy: latency-based-routing
-  routing_strategy_args:
-    # only assign 40% of traffic to the fastest deployment to avoid overloading it
-    lowest_latency_buffer: 0.4
+# router_settings:
+#   routing_strategy: latency-based-routing
+#   routing_strategy_args:
+#     # only assign 40% of traffic to the fastest deployment to avoid overloading it
+#     lowest_latency_buffer: 0.4
 
-    # consider last five minutes of calls for latency calculation
-    ttl: 300
-  redis_host: os.environ/REDIS_HOST
-  redis_port: os.environ/REDIS_PORT
-  redis_password: os.environ/REDIS_PASSWORD
+#     # consider last five minutes of calls for latency calculation
+#     ttl: 300
+#   redis_host: os.environ/REDIS_HOST
+#   redis_port: os.environ/REDIS_PORT
+#   redis_password: os.environ/REDIS_PASSWORD
 
-# see https://docs.litellm.ai/docs/proxy/prod#1-use-this-configyaml
-general_settings:
-  master_key: os.environ/LITELLM_MASTER_KEY
-  database_url: os.environ/DATABASE_URL
-  disable_master_key_return: true
-  # alerting: ['slack', 'email']
-  alerting: ['email']
+# # see https://docs.litellm.ai/docs/proxy/prod#1-use-this-configyaml
+# general_settings:
+#   master_key: os.environ/LITELLM_MASTER_KEY
+#   database_url: os.environ/DATABASE_URL
+#   disable_master_key_return: true
+#   # alerting: ['slack', 'email']
+#   alerting: ['email']
 
-  # Batch write spend updates every 60s
-  proxy_batch_write_at: 60
+#   # Batch write spend updates every 60s
+#   proxy_batch_write_at: 60
 
-  # see https://docs.litellm.ai/docs/proxy/caching#advanced---user-api-key-cache-ttl
-  # our api keys rarely change
-  user_api_key_cache_ttl: 3600
+#   # see https://docs.litellm.ai/docs/proxy/caching#advanced---user-api-key-cache-ttl
+#   # our api keys rarely change
+#   user_api_key_cache_ttl: 3600

litellm/router_utils/pattern_match_deployments.py

@@ -61,6 +61,24 @@ class PatternMatchRouter:
         # return f"^{regex}$"
         return re.escape(pattern).replace(r"\*", "(.*)")
 
+    def return_pattern_matched_deployments(
+        self, matched_pattern: Match, deployments: List[Dict]
+    ) -> List[Dict]:
+        new_deployments = []
+        for deployment in deployments:
+            new_deployment = copy.deepcopy(deployment)
+            new_deployment["litellm_params"]["model"] = (
+                PatternMatchRouter.set_deployment_model_name(
+                    matched_pattern=matched_pattern,
+                    litellm_deployment_litellm_model=deployment["litellm_params"][
+                        "model"
+                    ],
+                )
+            )
+            new_deployments.append(new_deployment)
+
+        return new_deployments
+
     def route(self, request: Optional[str]) -> Optional[List[Dict]]:
         """
         Route a requested model to the corresponding llm deployments based on the regex pattern
@@ -79,8 +97,11 @@ class PatternMatchRouter:
             if request is None:
                 return None
             for pattern, llm_deployments in self.patterns.items():
-                if re.match(pattern, request):
-                    return llm_deployments
+                pattern_match = re.match(pattern, request)
+                if pattern_match:
+                    return self.return_pattern_matched_deployments(
+                        matched_pattern=pattern_match, deployments=llm_deployments
+                    )
         except Exception as e:
             verbose_router_logger.debug(f"Error in PatternMatchRouter.route: {str(e)}")
 
@@ -102,6 +123,7 @@ class PatternMatchRouter:
 
         if model_name = "llmengine/foo" -> model = "openai/foo"
         """
+
         ## BASE CASE: if the deployment model name does not contain a wildcard, return the deployment model name
         if "*" not in litellm_deployment_litellm_model:
             return litellm_deployment_litellm_model
@@ -165,12 +187,7 @@ class PatternMatchRouter:
         """
         pattern_match = self.get_pattern(model, custom_llm_provider)
         if pattern_match:
-            provider_deployments = []
-            for deployment in pattern_match:
-                dep = copy.deepcopy(deployment)
-                dep["litellm_params"]["model"] = model
-                provider_deployments.append(dep)
-            return provider_deployments
+            return pattern_match
         return []
 
 

tests/local_testing/test_router_pattern_matching.py

@@ -17,6 +17,7 @@ from litellm.router import Deployment, LiteLLM_Params, ModelInfo
 from concurrent.futures import ThreadPoolExecutor
 from collections import defaultdict
 from dotenv import load_dotenv
+from unittest.mock import patch, MagicMock, AsyncMock
 
 load_dotenv()
 
@@ -155,3 +156,35 @@ def test_route_with_exception():
 
     result = router.route("openai/gpt-3.5-turbo")
     assert result is None
+
+
+def test_router_pattern_match_e2e():
+    """
+    Tests the end to end flow of the router
+    """
+    from litellm.llms.custom_httpx.http_handler import HTTPHandler
+
+    client = HTTPHandler()
+    router = Router(
+        model_list=[
+            {
+                "model_name": "llmengine/*",
+                "litellm_params": {"model": "anthropic/*", "api_key": "test"},
+            }
+        ]
+    )
+
+    with patch.object(client, "post", new=MagicMock()) as mock_post:
+
+        router.completion(
+            model="llmengine/my-custom-model",
+            messages=[{"role": "user", "content": "Hello, how are you?"}],
+            client=client,
+            api_key="test",
+        )
+        mock_post.assert_called_once()
+        print(mock_post.call_args.kwargs["data"])
+        mock_post.call_args.kwargs["data"] == {
+            "model": "gpt-4o",
+            "messages": [{"role": "user", "content": "Hello, how are you?"}],
+        }