From e67daf79be891448a278001b8d2637e1ed345af0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Jul 2024 18:22:35 -0700 Subject: [PATCH 1/8] router support setting pass_through_all_models --- litellm/types/router.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/types/router.py b/litellm/types/router.py index 78dfbc4c195..285732121c0 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -540,3 +540,6 @@ class RouterGeneralSettings(BaseModel): async_only_mode: bool = Field( default=False ) # this will only initialize async clients. Good for memory utils + pass_through_all_models: bool = Field( + default=False + ) # if passed a model not llm_router model list, pass through the request to litellm.acompletion/embedding From 8f4c5437b8bf7a1fa4501c305e68ace752ab73ea Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Jul 2024 18:34:12 -0700 Subject: [PATCH 2/8] router support setting pass_through_all_models --- litellm/proxy/proxy_server.py | 15 +++++++++++++++ litellm/router.py | 14 ++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f22f25f732d..022bb3040f2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2885,6 +2885,11 @@ async def chat_completion( and llm_router.default_deployment is not None ): # model in router deployments, calling a specific deployment on the router tasks.append(llm_router.acompletion(**data)) + elif ( + llm_router is not None + and llm_router.router_general_settings.pass_through_all_models is True + ): + tasks.append(litellm.acompletion(**data)) elif user_model is not None: # `litellm --model ` tasks.append(litellm.acompletion(**data)) else: @@ -3147,6 +3152,11 @@ async def completion( llm_response = asyncio.create_task(llm_router.atext_completion(**data)) elif user_model is not None: # `litellm --model ` llm_response = asyncio.create_task(litellm.atext_completion(**data)) + elif ( + llm_router is not None + and llm_router.router_general_settings.pass_through_all_models is True + ): + llm_response = asyncio.create_task(litellm.atext_completion(**data)) else: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, @@ -3405,6 +3415,11 @@ async def embeddings( and llm_router.default_deployment is not None ): # model in router deployments, calling a specific deployment on the router tasks.append(llm_router.aembedding(**data)) + elif ( + llm_router is not None + and llm_router.router_general_settings.pass_through_all_models is True + ): + tasks.append(litellm.aembedding(**data)) elif user_model is not None: # `litellm --model ` tasks.append(litellm.aembedding(**data)) else: diff --git a/litellm/router.py b/litellm/router.py index 53013a75941..d1198aa154e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -174,7 +174,9 @@ class Router: routing_strategy_args: dict = {}, # just for latency-based routing semaphore: Optional[asyncio.Semaphore] = None, alerting_config: Optional[AlertingConfig] = None, - router_general_settings: Optional[RouterGeneralSettings] = None, + router_general_settings: Optional[ + RouterGeneralSettings + ] = RouterGeneralSettings(), ) -> None: """ Initialize the Router class with the given parameters for caching, reliability, and routing strategy. @@ -253,8 +255,8 @@ class Router: verbose_router_logger.setLevel(logging.INFO) elif debug_level == "DEBUG": verbose_router_logger.setLevel(logging.DEBUG) - self.router_general_settings: Optional[RouterGeneralSettings] = ( - router_general_settings + self.router_general_settings: RouterGeneralSettings = ( + router_general_settings or RouterGeneralSettings() ) self.assistants_config = assistants_config @@ -3554,7 +3556,11 @@ class Router: # Check if user is trying to use model_name == "*" # this is a catch all model for their specific api key if deployment.model_name == "*": - self.default_deployment = deployment.to_json(exclude_none=True) + if deployment.litellm_params.model == "*": + # user wants to pass through all requests to litellm.acompletion for unknown deployments + self.router_general_settings.pass_through_all_models = True + else: + self.default_deployment = deployment.to_json(exclude_none=True) # Azure GPT-Vision Enhancements, users can pass os.environ/ data_sources = deployment.litellm_params.get("dataSources", []) or [] From 35203cede7c87433ef14ebd4f0ae9c14da363320 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Jul 2024 18:40:04 -0700 Subject: [PATCH 3/8] add ANTHROPIC_API_KEY on build and test --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e3593e81544..a9a5be67193 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -208,6 +208,7 @@ jobs: -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e AUTO_INFER_REGION=True \ From 986352037645d688e5e1c78dc3958ee459a24fae Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Jul 2024 18:48:56 -0700 Subject: [PATCH 4/8] support using */* --- .circleci/config.yml | 2 +- litellm/proxy/proxy_config.yaml | 3 +++ litellm/tests/test_get_llm_provider.py | 5 +++++ litellm/utils.py | 2 ++ proxy_server_config.yaml | 7 +++++++ 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a9a5be67193..a29b76110c3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -208,7 +208,7 @@ jobs: -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ -e MISTRAL_API_KEY=$MISTRAL_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e GROQ_API_KEY=$GROQ_API_KEY \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e AUTO_INFER_REGION=True \ diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 0e3f0826e27..9d913b45875 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -8,6 +8,9 @@ model_list: litellm_params: model: fireworks_ai/accounts/fireworks/models/llama-v3-70b-instruct api_key: "os.environ/FIREWORKS" + - model_name: "*" + litellm_params: + model: "*" general_settings: master_key: sk-1234 alerting: ["slack"] diff --git a/litellm/tests/test_get_llm_provider.py b/litellm/tests/test_get_llm_provider.py index 3ec867af444..6f53b0f8fec 100644 --- a/litellm/tests/test_get_llm_provider.py +++ b/litellm/tests/test_get_llm_provider.py @@ -25,6 +25,11 @@ def test_get_llm_provider(): # test_get_llm_provider() +def test_get_llm_provider_catch_all(): + _, response, _, _ = litellm.get_llm_provider(model="*") + assert response == "openai" + + def test_get_llm_provider_gpt_instruct(): _, response, _, _ = litellm.get_llm_provider(model="gpt-3.5-turbo-instruct-0914") diff --git a/litellm/utils.py b/litellm/utils.py index e104de958a8..cceed6b9d34 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4650,6 +4650,8 @@ def get_llm_provider( custom_llm_provider = "openai" elif model in litellm.empower_models: custom_llm_provider = "empower" + elif model == "*": + custom_llm_provider = "openai" if custom_llm_provider is None or custom_llm_provider == "": if litellm.suppress_debug_info == False: print() # noqa diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index 5ee7192c88d..f7766b65bfe 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -85,6 +85,13 @@ model_list: litellm_params: model: openai/* api_key: os.environ/OPENAI_API_KEY + + # Pass through all llm requests to litellm.completion/litellm.embedding + # if user passes model="anthropic/claude-3-opus-20240229" proxy will make requests to anthropic claude-3-opus-20240229 using ANTHROPIC_API_KEY + - model_name: "*" + litellm_params: + model: "*" + - model_name: mistral-embed litellm_params: model: mistral/mistral-embed From 05858cb249bdd594a9dd084b6d71bf79b9449199 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Jul 2024 18:54:30 -0700 Subject: [PATCH 5/8] test proxy all model --- tests/test_openai_endpoints.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index 59ac1055262..a77da8d52ca 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -7,6 +7,9 @@ from openai import OpenAI, AsyncOpenAI from typing import Optional, List, Union +LITELLM_MASTER_KEY = "sk-1234" + + def response_header_check(response): """ - assert if response headers < 4kb (nginx limit). @@ -467,6 +470,22 @@ async def test_openai_wildcard_chat_completion(): await chat_completion(session=session, key=key, model="gpt-3.5-turbo-0125") +@pytest.mark.asyncio +async def test_proxy_all_models(): + """ + - proxy_server_config.yaml has model = * / * + - Make chat completion call + - groq is NOT defined on /models + + + """ + async with aiohttp.ClientSession() as session: + # call chat/completions with a model that the key was not created for + the model is not on the config.yaml + await chat_completion( + session=session, key=LITELLM_MASTER_KEY, model="groq/llama3-8b-8192" + ) + + @pytest.mark.asyncio async def test_batch_chat_completions(): """ From ff0f21a1f35f0bf1c7f5c2372881291b4761defb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Jul 2024 19:02:22 -0700 Subject: [PATCH 6/8] docs - anthropic --- docs/my-website/docs/providers/anthropic.md | 41 +++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 496343f8792..2227b7a6b51 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -82,6 +82,47 @@ model_list: ```bash litellm --config /path/to/config.yaml ``` + + + +Use this if you want to make requests to `claude-3-haiku-20240307`,`claude-3-opus-20240229`,`claude-2.1` without defining them on the config.yaml + +#### Required env variables +``` +ANTHROPIC_API_KEY=sk-ant**** +``` + +```yaml +model_list: + - model_name: "*" + litellm_params: + model: "*" +``` + +```bash +litellm --config /path/to/config.yaml +``` + +Example Request for this config.yaml + +**Ensure you use `anthropic/` prefix to route the request to Anthropic API** + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data ' { + "model": "anthropic/claude-3-haiku-20240307", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + From af1cd9e06f900e8bba35d4083847cf2e2b97f60a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Jul 2024 19:17:20 -0700 Subject: [PATCH 7/8] docs on pass through support --- docs/my-website/docs/proxy/configs.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index ecd82375e55..cb0841c60c5 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -59,6 +59,13 @@ model_list: rpm: 1440 model_info: version: 2 + + # Use this if you want to make requests to `claude-3-haiku-20240307`,`claude-3-opus-20240229`,`claude-2.1` without defining them on the config.yaml + # Default models + # Works for ALL Providers and needs the default provider credentials in .env + - model_name: "*" + litellm_params: + model: "*" litellm_settings: # module level litellm settings - https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py drop_params: True From 693bcfac39d6f930e0dc14a2d233bb158f2ac65e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 25 Jul 2024 19:32:49 -0700 Subject: [PATCH 8/8] fix using pass_through_all_models --- litellm/proxy/proxy_config.yaml | 4 ++++ litellm/proxy/proxy_server.py | 33 ++++++++++++++++++--------------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 9d913b45875..bb256c49ba2 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -11,6 +11,10 @@ model_list: - model_name: "*" litellm_params: model: "*" + - model_name: "*" + litellm_params: + model: openai/* + api_key: os.environ/OPENAI_API_KEY general_settings: master_key: sk-1234 alerting: ["slack"] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 022bb3040f2..1c9a36912da 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2882,14 +2882,15 @@ async def chat_completion( elif ( llm_router is not None and data["model"] not in router_model_names - and llm_router.default_deployment is not None - ): # model in router deployments, calling a specific deployment on the router - tasks.append(llm_router.acompletion(**data)) - elif ( - llm_router is not None and llm_router.router_general_settings.pass_through_all_models is True ): tasks.append(litellm.acompletion(**data)) + elif ( + llm_router is not None + and data["model"] not in router_model_names + and llm_router.default_deployment is not None + ): # model in router deployments, calling a specific deployment on the router + tasks.append(llm_router.acompletion(**data)) elif user_model is not None: # `litellm --model ` tasks.append(litellm.acompletion(**data)) else: @@ -3144,6 +3145,12 @@ async def completion( llm_router is not None and data["model"] in llm_router.get_model_ids() ): # model in router model list llm_response = asyncio.create_task(llm_router.atext_completion(**data)) + elif ( + llm_router is not None + and data["model"] not in router_model_names + and llm_router.router_general_settings.pass_through_all_models is True + ): + llm_response = asyncio.create_task(litellm.atext_completion(**data)) elif ( llm_router is not None and data["model"] not in router_model_names @@ -3152,11 +3159,6 @@ async def completion( llm_response = asyncio.create_task(llm_router.atext_completion(**data)) elif user_model is not None: # `litellm --model ` llm_response = asyncio.create_task(litellm.atext_completion(**data)) - elif ( - llm_router is not None - and llm_router.router_general_settings.pass_through_all_models is True - ): - llm_response = asyncio.create_task(litellm.atext_completion(**data)) else: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, @@ -3412,14 +3414,15 @@ async def embeddings( elif ( llm_router is not None and data["model"] not in router_model_names - and llm_router.default_deployment is not None - ): # model in router deployments, calling a specific deployment on the router - tasks.append(llm_router.aembedding(**data)) - elif ( - llm_router is not None and llm_router.router_general_settings.pass_through_all_models is True ): tasks.append(litellm.aembedding(**data)) + elif ( + llm_router is not None + and data["model"] not in router_model_names + and llm_router.default_deployment is not None + ): # model in router deployments, calling a specific deployment on the router + tasks.append(llm_router.aembedding(**data)) elif user_model is not None: # `litellm --model ` tasks.append(litellm.aembedding(**data)) else: