diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index b9315670a2c..acbac00107a 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -13,6 +13,10 @@ model_list: - model_name: gemini-1.5-flash-002 litellm_params: model: gemini/gemini-1.5-flash-002 + - model_name: "llmengine/*" + litellm_params: + model: "openai/*" + api_key: sk-1234 # litellm_settings: # fallbacks: [{ "claude-3-5-sonnet-20240620": ["claude-3-5-sonnet-aihubmix"] }] @@ -20,53 +24,53 @@ model_list: # default_redis_batch_cache_expiry: 10 -litellm_settings: - cache: True - cache_params: - type: redis +# litellm_settings: +# cache: True +# cache_params: +# type: redis - # disable caching on the actual API call - supported_call_types: [] +# # disable caching on the actual API call +# supported_call_types: [] - # see https://docs.litellm.ai/docs/proxy/prod#3-use-redis-porthost-password-not-redis_url - host: os.environ/REDIS_HOST - port: os.environ/REDIS_PORT - password: os.environ/REDIS_PASSWORD +# # see https://docs.litellm.ai/docs/proxy/prod#3-use-redis-porthost-password-not-redis_url +# host: os.environ/REDIS_HOST +# port: os.environ/REDIS_PORT +# password: os.environ/REDIS_PASSWORD # see https://docs.litellm.ai/docs/proxy/caching#turn-on-batch_redis_requests # see https://docs.litellm.ai/docs/proxy/prometheus - callbacks: ['prometheus', 'otel'] + # callbacks: ['prometheus', 'otel'] - # # see https://docs.litellm.ai/docs/proxy/logging#logging-proxy-inputoutput---sentry - failure_callback: ['sentry'] - service_callback: ['prometheus_system'] + # # # see https://docs.litellm.ai/docs/proxy/logging#logging-proxy-inputoutput---sentry + # failure_callback: ['sentry'] + # service_callback: ['prometheus_system'] # redact_user_api_key_info: true -router_settings: - routing_strategy: latency-based-routing - routing_strategy_args: - # only assign 40% of traffic to the fastest deployment to avoid overloading it - lowest_latency_buffer: 0.4 +# router_settings: +# routing_strategy: latency-based-routing +# routing_strategy_args: +# # only assign 40% of traffic to the fastest deployment to avoid overloading it +# lowest_latency_buffer: 0.4 - # consider last five minutes of calls for latency calculation - ttl: 300 - redis_host: os.environ/REDIS_HOST - redis_port: os.environ/REDIS_PORT - redis_password: os.environ/REDIS_PASSWORD +# # consider last five minutes of calls for latency calculation +# ttl: 300 +# redis_host: os.environ/REDIS_HOST +# redis_port: os.environ/REDIS_PORT +# redis_password: os.environ/REDIS_PASSWORD -# see https://docs.litellm.ai/docs/proxy/prod#1-use-this-configyaml -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - database_url: os.environ/DATABASE_URL - disable_master_key_return: true - # alerting: ['slack', 'email'] - alerting: ['email'] +# # see https://docs.litellm.ai/docs/proxy/prod#1-use-this-configyaml +# general_settings: +# master_key: os.environ/LITELLM_MASTER_KEY +# database_url: os.environ/DATABASE_URL +# disable_master_key_return: true +# # alerting: ['slack', 'email'] +# alerting: ['email'] - # Batch write spend updates every 60s - proxy_batch_write_at: 60 +# # Batch write spend updates every 60s +# proxy_batch_write_at: 60 - # see https://docs.litellm.ai/docs/proxy/caching#advanced---user-api-key-cache-ttl - # our api keys rarely change - user_api_key_cache_ttl: 3600 \ No newline at end of file +# # see https://docs.litellm.ai/docs/proxy/caching#advanced---user-api-key-cache-ttl +# # our api keys rarely change +# user_api_key_cache_ttl: 3600 \ No newline at end of file diff --git a/litellm/router_utils/pattern_match_deployments.py b/litellm/router_utils/pattern_match_deployments.py index a0d631bf71d..c6749c98c7b 100644 --- a/litellm/router_utils/pattern_match_deployments.py +++ b/litellm/router_utils/pattern_match_deployments.py @@ -61,6 +61,24 @@ class PatternMatchRouter: # return f"^{regex}$" return re.escape(pattern).replace(r"\*", "(.*)") + def return_pattern_matched_deployments( + self, matched_pattern: Match, deployments: List[Dict] + ) -> List[Dict]: + new_deployments = [] + for deployment in deployments: + new_deployment = copy.deepcopy(deployment) + new_deployment["litellm_params"]["model"] = ( + PatternMatchRouter.set_deployment_model_name( + matched_pattern=matched_pattern, + litellm_deployment_litellm_model=deployment["litellm_params"][ + "model" + ], + ) + ) + new_deployments.append(new_deployment) + + return new_deployments + def route(self, request: Optional[str]) -> Optional[List[Dict]]: """ Route a requested model to the corresponding llm deployments based on the regex pattern @@ -79,8 +97,11 @@ class PatternMatchRouter: if request is None: return None for pattern, llm_deployments in self.patterns.items(): - if re.match(pattern, request): - return llm_deployments + pattern_match = re.match(pattern, request) + if pattern_match: + return self.return_pattern_matched_deployments( + matched_pattern=pattern_match, deployments=llm_deployments + ) except Exception as e: verbose_router_logger.debug(f"Error in PatternMatchRouter.route: {str(e)}") @@ -102,6 +123,7 @@ class PatternMatchRouter: if model_name = "llmengine/foo" -> model = "openai/foo" """ + ## BASE CASE: if the deployment model name does not contain a wildcard, return the deployment model name if "*" not in litellm_deployment_litellm_model: return litellm_deployment_litellm_model @@ -165,12 +187,7 @@ class PatternMatchRouter: """ pattern_match = self.get_pattern(model, custom_llm_provider) if pattern_match: - provider_deployments = [] - for deployment in pattern_match: - dep = copy.deepcopy(deployment) - dep["litellm_params"]["model"] = model - provider_deployments.append(dep) - return provider_deployments + return pattern_match return [] diff --git a/tests/local_testing/test_router_pattern_matching.py b/tests/local_testing/test_router_pattern_matching.py index 701a62e4127..9d8c4db0dac 100644 --- a/tests/local_testing/test_router_pattern_matching.py +++ b/tests/local_testing/test_router_pattern_matching.py @@ -17,6 +17,7 @@ from litellm.router import Deployment, LiteLLM_Params, ModelInfo from concurrent.futures import ThreadPoolExecutor from collections import defaultdict from dotenv import load_dotenv +from unittest.mock import patch, MagicMock, AsyncMock load_dotenv() @@ -155,3 +156,35 @@ def test_route_with_exception(): result = router.route("openai/gpt-3.5-turbo") assert result is None + + +def test_router_pattern_match_e2e(): + """ + Tests the end to end flow of the router + """ + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + router = Router( + model_list=[ + { + "model_name": "llmengine/*", + "litellm_params": {"model": "anthropic/*", "api_key": "test"}, + } + ] + ) + + with patch.object(client, "post", new=MagicMock()) as mock_post: + + router.completion( + model="llmengine/my-custom-model", + messages=[{"role": "user", "content": "Hello, how are you?"}], + client=client, + api_key="test", + ) + mock_post.assert_called_once() + print(mock_post.call_args.kwargs["data"]) + mock_post.call_args.kwargs["data"] == { + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Hello, how are you?"}], + }