Merge pull request #4891 from BerriAI/litellm_proxy_support_all_providers

[Feat] Support /* for multiple providers

.circleci/config.yml

@@ -208,6 +208,7 @@ jobs:
               -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
               -e MISTRAL_API_KEY=$MISTRAL_API_KEY \
               -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+              -e GROQ_API_KEY=$GROQ_API_KEY \
               -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
               -e AWS_REGION_NAME=$AWS_REGION_NAME \
               -e AUTO_INFER_REGION=True \

docs/my-website/docs/providers/anthropic.md

@@ -82,6 +82,47 @@ model_list:
 ```bash
 litellm --config /path/to/config.yaml
 ```
+</TabItem>
+<TabItem value="config-all" label="config - default all Anthropic Model">
+
+Use this if you want to make requests to `claude-3-haiku-20240307`,`claude-3-opus-20240229`,`claude-2.1` without defining them on the config.yaml
+
+#### Required env variables
+```
+ANTHROPIC_API_KEY=sk-ant****
+```
+
+```yaml
+model_list:
+  - model_name: "*" 
+    litellm_params:
+      model: "*"
+```
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+Example Request for this config.yaml
+
+**Ensure you use `anthropic/` prefix to route the request to Anthropic API**
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--data ' {
+      "model": "anthropic/claude-3-haiku-20240307",
+      "messages": [
+        {
+          "role": "user",
+          "content": "what llm are you"
+        }
+      ]
+    }
+'
+```
+
+
 </TabItem>
 <TabItem value="cli" label="cli">
 

docs/my-website/docs/proxy/configs.md

@@ -59,6 +59,13 @@ model_list:
       rpm: 1440
     model_info: 
       version: 2
+  
+  # Use this if you want to make requests to `claude-3-haiku-20240307`,`claude-3-opus-20240229`,`claude-2.1` without defining them on the config.yaml
+  # Default models
+  # Works for ALL Providers and needs the default provider credentials in .env
+  - model_name: "*" 
+    litellm_params:
+      model: "*"
 
 litellm_settings: # module level litellm settings - https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py
   drop_params: True

litellm/proxy/proxy_config.yaml

@@ -8,6 +8,13 @@ model_list:
     litellm_params:
       model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct
       api_key: "os.environ/FIREWORKS"
+  - model_name: "*"
+    litellm_params:
+      model: "*"
+  - model_name: "*"
+    litellm_params:
+      model: openai/*
+      api_key: os.environ/OPENAI_API_KEY
   - model_name: mistral-small-latest
     litellm_params:
       model: mistral/mistral-small-latest

litellm/proxy/proxy_server.py

@@ -2894,6 +2894,12 @@ async def chat_completion(
             llm_router is not None and data["model"] in llm_router.deployment_names
         ):  # model in router deployments, calling a specific deployment on the router
             tasks.append(llm_router.acompletion(**data, specific_deployment=True))
+        elif (
+            llm_router is not None
+            and data["model"] not in router_model_names
+            and llm_router.router_general_settings.pass_through_all_models is True
+        ):
+            tasks.append(litellm.acompletion(**data))
         elif (
             llm_router is not None
             and data["model"] not in router_model_names
@@ -3154,6 +3160,12 @@ async def completion(
             llm_router is not None and data["model"] in llm_router.get_model_ids()
         ):  # model in router model list
             llm_response = asyncio.create_task(llm_router.atext_completion(**data))
+        elif (
+            llm_router is not None
+            and data["model"] not in router_model_names
+            and llm_router.router_general_settings.pass_through_all_models is True
+        ):
+            llm_response = asyncio.create_task(litellm.atext_completion(**data))
         elif (
             llm_router is not None
             and data["model"] not in router_model_names
@@ -3414,6 +3426,12 @@ async def embeddings(
             llm_router is not None and data["model"] in llm_router.get_model_ids()
         ):  # model in router deployments, calling a specific deployment on the router
             tasks.append(llm_router.aembedding(**data))
+        elif (
+            llm_router is not None
+            and data["model"] not in router_model_names
+            and llm_router.router_general_settings.pass_through_all_models is True
+        ):
+            tasks.append(litellm.aembedding(**data))
         elif (
             llm_router is not None
             and data["model"] not in router_model_names

litellm/router.py

@@ -174,7 +174,9 @@ class Router:
         routing_strategy_args: dict = {},  # just for latency-based routing
         semaphore: Optional[asyncio.Semaphore] = None,
         alerting_config: Optional[AlertingConfig] = None,
-        router_general_settings: Optional[RouterGeneralSettings] = None,
+        router_general_settings: Optional[
+            RouterGeneralSettings
+        ] = RouterGeneralSettings(),
     ) -> None:
         """
         Initialize the Router class with the given parameters for caching, reliability, and routing strategy.
@@ -253,8 +255,8 @@ class Router:
                 verbose_router_logger.setLevel(logging.INFO)
             elif debug_level == "DEBUG":
                 verbose_router_logger.setLevel(logging.DEBUG)
-        self.router_general_settings: Optional[RouterGeneralSettings] = (
-            router_general_settings
+        self.router_general_settings: RouterGeneralSettings = (
+            router_general_settings or RouterGeneralSettings()
         )
 
         self.assistants_config = assistants_config
@@ -3554,7 +3556,11 @@ class Router:
         # Check if user is trying to use model_name == "*"
         # this is a catch all model for their specific api key
         if deployment.model_name == "*":
-            self.default_deployment = deployment.to_json(exclude_none=True)
+            if deployment.litellm_params.model == "*":
+                # user wants to pass through all requests to litellm.acompletion for unknown deployments
+                self.router_general_settings.pass_through_all_models = True
+            else:
+                self.default_deployment = deployment.to_json(exclude_none=True)
 
         # Azure GPT-Vision Enhancements, users can pass os.environ/
         data_sources = deployment.litellm_params.get("dataSources", []) or []

litellm/tests/test_get_llm_provider.py

@@ -25,6 +25,11 @@ def test_get_llm_provider():
 # test_get_llm_provider()
 
 
+def test_get_llm_provider_catch_all():
+    _, response, _, _ = litellm.get_llm_provider(model="*")
+    assert response == "openai"
+
+
 def test_get_llm_provider_gpt_instruct():
     _, response, _, _ = litellm.get_llm_provider(model="gpt-3.5-turbo-instruct-0914")
 

litellm/types/router.py

@@ -540,3 +540,6 @@ class RouterGeneralSettings(BaseModel):
     async_only_mode: bool = Field(
         default=False
     )  # this will only initialize async clients. Good for memory utils
+    pass_through_all_models: bool = Field(
+        default=False
+    )  # if passed a model not llm_router model list, pass through the request to litellm.acompletion/embedding

litellm/utils.py

@@ -4667,6 +4667,8 @@ def get_llm_provider(
             custom_llm_provider = "openai"
         elif model in litellm.empower_models:
             custom_llm_provider = "empower"
+        elif model == "*":
+            custom_llm_provider = "openai"
         if custom_llm_provider is None or custom_llm_provider == "":
             if litellm.suppress_debug_info == False:
                 print()  # noqa

proxy_server_config.yaml

@@ -85,6 +85,13 @@ model_list:
     litellm_params:
       model: openai/*
       api_key: os.environ/OPENAI_API_KEY
+  
+  # Pass through all llm requests to litellm.completion/litellm.embedding
+  # if user passes model="anthropic/claude-3-opus-20240229" proxy will make requests to anthropic claude-3-opus-20240229 using ANTHROPIC_API_KEY
+  - model_name: "*"
+    litellm_params:
+      model: "*"
+  
   - model_name: mistral-embed
     litellm_params:
       model: mistral/mistral-embed

tests/test_openai_endpoints.py

@@ -7,6 +7,9 @@ from openai import OpenAI, AsyncOpenAI
 from typing import Optional, List, Union
 
 
+LITELLM_MASTER_KEY = "sk-1234"
+
+
 def response_header_check(response):
     """
     - assert if response headers < 4kb (nginx limit).
@@ -467,6 +470,22 @@ async def test_openai_wildcard_chat_completion():
         await chat_completion(session=session, key=key, model="gpt-3.5-turbo-0125")
 
 
+@pytest.mark.asyncio
+async def test_proxy_all_models():
+    """
+    - proxy_server_config.yaml has model = * / *
+    - Make chat completion call
+    - groq is NOT defined on /models
+
+
+    """
+    async with aiohttp.ClientSession() as session:
+        # call chat/completions with a model that the key was not created for + the model is not on the config.yaml
+        await chat_completion(
+            session=session, key=LITELLM_MASTER_KEY, model="groq/llama3-8b-8192"
+        )
+
+
 @pytest.mark.asyncio
 async def test_batch_chat_completions():
     """