From 18305b23f429c1c9e6c9fed123497a40442f2b15 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 13:49:46 -0700 Subject: [PATCH 1/7] add + test provider specific routing --- litellm/router.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 5a4d83885f9..51fb12ea877 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -17,6 +17,7 @@ import inspect import json import logging import random +import re import threading import time import traceback @@ -310,6 +311,7 @@ class Router: ) self.default_deployment = None # use this to track the users default deployment, when they want to use model = * self.default_max_parallel_requests = default_max_parallel_requests + self.provider_default_deployments: Dict[str, List] = {} if model_list is not None: model_list = copy.deepcopy(model_list) @@ -3607,6 +3609,10 @@ class Router: ), ) + provider_specific_deployment = re.match( + f"{custom_llm_provider}/*", deployment.model_name + ) + # Check if user is trying to use model_name == "*" # this is a catch all model for their specific api key if deployment.model_name == "*": @@ -3615,6 +3621,17 @@ class Router: self.router_general_settings.pass_through_all_models = True else: self.default_deployment = deployment.to_json(exclude_none=True) + # Check if user is using provider specific wildcard routing + # example model_name = "databricks/*" or model_name = "anthropic/*" + elif provider_specific_deployment: + if custom_llm_provider in self.provider_default_deployments: + self.provider_default_deployments[custom_llm_provider].append( + deployment.to_json(exclude_none=True) + ) + else: + self.provider_default_deployments[custom_llm_provider] = [ + deployment.to_json(exclude_none=True) + ] # Azure GPT-Vision Enhancements, users can pass os.environ/ data_sources = deployment.litellm_params.get("dataSources", []) or [] From 3249e295cb9817ac11ea3e0842821a0eb6045469 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 13:52:00 -0700 Subject: [PATCH 2/7] test provider wildcard routing --- litellm/tests/test_router.py | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/litellm/tests/test_router.py b/litellm/tests/test_router.py index 38f274d5646..98f4792e015 100644 --- a/litellm/tests/test_router.py +++ b/litellm/tests/test_router.py @@ -60,6 +60,64 @@ def test_router_multi_org_list(): assert len(router.get_model_list()) == 3 +@pytest.mark.asyncio() +async def test_router_provider_wildcard_routing(): + """ + Pass list of orgs in 1 model definition, + expect a unique deployment for each to be created + """ + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "my-key", + "api_base": "https://api.openai.com/v1", + "organization": ["org-1", "org-2", "org-3"], + }, + }, + { + "model_name": "anthropic/*", + "litellm_params": { + "model": "anthropic/*", + "api_key": "my-key", + }, + }, + { + "model_name": "databricks/*", + "litellm_params": { + "model": "databricks/*", + "api_key": "my-key", + }, + }, + ] + ) + + print("router model list = ", router.get_model_list()) + + response1 = await router.acompletion( + model="anthropic/claude-3-sonnet-20240229", + messages=[{"role": "user", "content": "hello"}], + ) + + print("response 1 = ", response1) + + response2 = await router.acompletion( + model="openai/gpt-3.5-turbo", + messages=[{"role": "user", "content": "hello"}], + ) + + print("response 2 = ", response2) + + response3 = await router.acompletion( + model="databricks/databricks-meta-llama-3-1-70b-instruct", + messages=[{"role": "user", "content": "hello"}], + ) + + print("response 3 = ", response3) + + def test_router_specific_model_via_id(): """ Call a specific deployment by it's id From 5d7a1b2ec6b0af1820031d7e1761bf9bd91fc047 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 14:12:10 -0700 Subject: [PATCH 3/7] router use provider specific wildcard routing --- litellm/router.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 51fb12ea877..9afd783227a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4475,6 +4475,29 @@ class Router: ) # self.default_deployment updated_deployment["litellm_params"]["model"] = model return model, updated_deployment + elif model not in self.model_names: + # check if provider/ specific wildcard routing + try: + ( + _, + custom_llm_provider, + _, + _, + ) = litellm.get_llm_provider(model=model) + # check if custom_llm_provider + if custom_llm_provider in self.provider_default_deployments: + _provider_deployments = self.provider_default_deployments[ + custom_llm_provider + ] + provider_deployments = [] + for deployment in _provider_deployments: + dep = copy.deepcopy(deployment) + dep["litellm_params"]["model"] = model + provider_deployments.append(dep) + return model, provider_deployments + except: + # get_llm_provider raises exception when provider is unknown + pass ## get healthy deployments ### get all deployments From d42949cb4aa9e602ce8a23369d0db28fb7d6f0d8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 14:12:40 -0700 Subject: [PATCH 4/7] test_router_provider_wildcard_routing --- litellm/tests/test_router.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/litellm/tests/test_router.py b/litellm/tests/test_router.py index 98f4792e015..12d485dde2d 100644 --- a/litellm/tests/test_router.py +++ b/litellm/tests/test_router.py @@ -72,23 +72,22 @@ async def test_router_provider_wildcard_routing(): "model_name": "openai/*", "litellm_params": { "model": "openai/*", - "api_key": "my-key", + "api_key": os.environ["OPENAI_API_KEY"], "api_base": "https://api.openai.com/v1", - "organization": ["org-1", "org-2", "org-3"], }, }, { "model_name": "anthropic/*", "litellm_params": { "model": "anthropic/*", - "api_key": "my-key", + "api_key": os.environ["ANTHROPIC_API_KEY"], }, }, { - "model_name": "databricks/*", + "model_name": "groq/*", "litellm_params": { - "model": "databricks/*", - "api_key": "my-key", + "model": "groq/*", + "api_key": os.environ["GROQ_API_KEY"], }, }, ] @@ -111,7 +110,7 @@ async def test_router_provider_wildcard_routing(): print("response 2 = ", response2) response3 = await router.acompletion( - model="databricks/databricks-meta-llama-3-1-70b-instruct", + model="groq/llama3-8b-8192", messages=[{"role": "user", "content": "hello"}], ) From 218ba0f4704ddc5ff15ecb6f32e5bae0e50d9969 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 14:20:22 -0700 Subject: [PATCH 5/7] support provider wildcard routing --- litellm/proxy/proxy_server.py | 40 ++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 29dc3813c65..299b390b9a0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3007,7 +3007,10 @@ async def chat_completion( elif ( llm_router is not None and data["model"] not in router_model_names - and llm_router.default_deployment is not None + and ( + llm_router.default_deployment is not None + or len(llm_router.provider_default_deployments) > 0 + ) ): # model in router deployments, calling a specific deployment on the router tasks.append(llm_router.acompletion(**data)) elif user_model is not None: # `litellm --model ` @@ -3275,7 +3278,10 @@ async def completion( elif ( llm_router is not None and data["model"] not in router_model_names - and llm_router.default_deployment is not None + and ( + llm_router.default_deployment is not None + or len(llm_router.provider_default_deployments) > 0 + ) ): # model in router deployments, calling a specific deployment on the router llm_response = asyncio.create_task(llm_router.atext_completion(**data)) elif user_model is not None: # `litellm --model ` @@ -3541,7 +3547,10 @@ async def embeddings( elif ( llm_router is not None and data["model"] not in router_model_names - and llm_router.default_deployment is not None + and ( + llm_router.default_deployment is not None + or len(llm_router.provider_default_deployments) > 0 + ) ): # model in router deployments, calling a specific deployment on the router tasks.append(llm_router.aembedding(**data)) elif user_model is not None: # `litellm --model ` @@ -3708,7 +3717,10 @@ async def image_generation( elif ( llm_router is not None and data["model"] not in router_model_names - and llm_router.default_deployment is not None + and ( + llm_router.default_deployment is not None + or len(llm_router.provider_default_deployments) > 0 + ) ): # model in router deployments, calling a specific deployment on the router response = await llm_router.aimage_generation(**data) elif user_model is not None: # `litellm --model ` @@ -3850,7 +3862,10 @@ async def audio_speech( elif ( llm_router is not None and data["model"] not in router_model_names - and llm_router.default_deployment is not None + and ( + llm_router.default_deployment is not None + or len(llm_router.provider_default_deployments) > 0 + ) ): # model in router deployments, calling a specific deployment on the router response = await llm_router.aspeech(**data) elif user_model is not None: # `litellm --model ` @@ -4020,7 +4035,10 @@ async def audio_transcriptions( elif ( llm_router is not None and data["model"] not in router_model_names - and llm_router.default_deployment is not None + and ( + llm_router.default_deployment is not None + or len(llm_router.provider_default_deployments) > 0 + ) ): # model in router deployments, calling a specific deployment on the router response = await llm_router.atranscription(**data) elif user_model is not None: # `litellm --model ` @@ -5270,7 +5288,10 @@ async def moderations( elif ( llm_router is not None and data.get("model") not in router_model_names - and llm_router.default_deployment is not None + and ( + llm_router.default_deployment is not None + or len(llm_router.provider_default_deployments) > 0 + ) ): # model in router deployments, calling a specific deployment on the router response = await llm_router.amoderation(**data) elif user_model is not None: # `litellm --model ` @@ -5421,7 +5442,10 @@ async def anthropic_response( elif ( llm_router is not None and data["model"] not in router_model_names - and llm_router.default_deployment is not None + and ( + llm_router.default_deployment is not None + or len(llm_router.provider_default_deployments) > 0 + ) ): # model in router deployments, calling a specific deployment on the router llm_response = asyncio.create_task(llm_router.aadapter_completion(**data)) elif user_model is not None: # `litellm --model ` From f1ffa820628bd4c7fa62427d08a9c16a1222cd22 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 14:37:20 -0700 Subject: [PATCH 6/7] fix use provider specific routing --- litellm/proxy/proxy_config.yaml | 10 ++++++++-- litellm/router.py | 16 +++++++++------- proxy_server_config.yaml | 14 +++++++++----- tests/test_openai_endpoints.py | 10 +++++++++- 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 36b191c90a5..d4bddd9a0ad 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -8,9 +8,15 @@ model_list: litellm_params: model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct api_key: "os.environ/FIREWORKS" - - model_name: "*" + # provider specific wildcard routing + - model_name: "anthropic/*" litellm_params: - model: "*" + model: "anthropic/*" + api_key: os.environ/ANTHROPIC_API_KEY + - model_name: "groq/*" + litellm_params: + model: "groq/*" + api_key: os.environ/GROQ_API_KEY - model_name: "*" litellm_params: model: openai/* diff --git a/litellm/router.py b/litellm/router.py index 9afd783227a..dc030d3690e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4469,13 +4469,7 @@ class Router: ) model = self.model_group_alias[model] - if model not in self.model_names and self.default_deployment is not None: - updated_deployment = copy.deepcopy( - self.default_deployment - ) # self.default_deployment - updated_deployment["litellm_params"]["model"] = model - return model, updated_deployment - elif model not in self.model_names: + if model not in self.model_names: # check if provider/ specific wildcard routing try: ( @@ -4499,6 +4493,14 @@ class Router: # get_llm_provider raises exception when provider is unknown pass + # check if default deployment is set + if self.default_deployment is not None: + updated_deployment = copy.deepcopy( + self.default_deployment + ) # self.default_deployment + updated_deployment["litellm_params"]["model"] = model + return model, updated_deployment + ## get healthy deployments ### get all deployments healthy_deployments = [m for m in self.model_list if m["model_name"] == model] diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index 4912ebbbfb1..57113d3509d 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -86,12 +86,16 @@ model_list: model: openai/* api_key: os.environ/OPENAI_API_KEY - # Pass through all llm requests to litellm.completion/litellm.embedding - # if user passes model="anthropic/claude-3-opus-20240229" proxy will make requests to anthropic claude-3-opus-20240229 using ANTHROPIC_API_KEY - - model_name: "*" + + # provider specific wildcard routing + - model_name: "anthropic/*" litellm_params: - model: "*" - + model: "anthropic/*" + api_key: os.environ/ANTHROPIC_API_KEY + - model_name: "groq/*" + litellm_params: + model: "groq/*" + api_key: os.environ/GROQ_API_KEY - model_name: mistral-embed litellm_params: model: mistral/mistral-embed diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index a77da8d52ca..932b32551f7 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -119,7 +119,9 @@ async def chat_completion(session, key, model: Union[str, List] = "gpt-4"): print() if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") + raise Exception( + f"Request did not return a 200 status code: {status}, response text={response_text}" + ) response_header_check( response @@ -485,6 +487,12 @@ async def test_proxy_all_models(): session=session, key=LITELLM_MASTER_KEY, model="groq/llama3-8b-8192" ) + await chat_completion( + session=session, + key=LITELLM_MASTER_KEY, + model="anthropic/claude-3-sonnet-20240229", + ) + @pytest.mark.asyncio async def test_batch_chat_completions(): From a367f97eb243f1bbac1b0fcd0149b55f2123b8b2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 7 Aug 2024 14:49:45 -0700 Subject: [PATCH 7/7] docs provider specific wildcard routing --- docs/my-website/docs/proxy/configs.md | 76 +++++++++++++++------------ 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 424ef8615bf..1620d11cad0 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -284,52 +284,58 @@ curl --location 'http://0.0.0.0:4000/v1/model/info' \ --data '' ``` -## Wildcard Model Name (Add ALL MODELS from env) + +## Provider specific wildcard routing +**Proxy all models from a provider** -Dynamically call any model from any given provider without the need to predefine it in the config YAML file. As long as the relevant keys are in the environment (see [providers list](../providers/)), LiteLLM will make the call correctly. +Use this if you want to **proxy all models from a specific provider without defining them on the config.yaml** - - -1. Setup config.yaml -``` +**Step 1** - define provider specific routing on config.yaml +```yaml model_list: - - model_name: "*" # all requests where model not in your config go to this deployment + # provider specific wildcard routing + - model_name: "anthropic/*" litellm_params: - model: "*" # passes our validation check that a real provider is given + model: "anthropic/*" + api_key: os.environ/ANTHROPIC_API_KEY + - model_name: "groq/*" + litellm_params: + model: "groq/*" + api_key: os.environ/GROQ_API_KEY ``` -2. Start LiteLLM proxy +Step 2 - Run litellm proxy -``` -litellm --config /path/to/config.yaml +```shell +$ litellm --config /path/to/config.yaml ``` -3. Try claude 3-5 sonnet from anthropic +Step 3 Test it -```bash -curl -X POST 'http://0.0.0.0:4000/chat/completions' \ --H 'Content-Type: application/json' \ --H 'Authorization: Bearer sk-1234' \ --D '{ - "model": "claude-3-5-sonnet-20240620", - "messages": [ - {"role": "user", "content": "Hey, how'\''s it going?"}, - { - "role": "assistant", - "content": "I'\''m doing well. Would like to hear the rest of the story?" - }, - {"role": "user", "content": "Na"}, - { - "role": "assistant", - "content": "No problem, is there anything else i can help you with today?" - }, - { - "role": "user", - "content": "I think you'\''re getting cut off sometimes" - } +Test with `anthropic/` - all models with `anthropic/` prefix will get routed to `anthropic/*` +```shell +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "anthropic/claude-3-sonnet-20240229", + "messages": [ + {"role": "user", "content": "Hello, Claude!"} ] -} -' + }' +``` + +Test with `groq/` - all models with `groq/` prefix will get routed to `groq/*` +```shell +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "groq/llama3-8b-8192", + "messages": [ + {"role": "user", "content": "Hello, Claude!"} + ] + }' ``` ## Load Balancing