From 65b1a7be73379060f626264622c44764b9bdd7bb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Apr 2024 08:51:39 -0700 Subject: [PATCH 01/10] add azure command_r_plust --- litellm/model_prices_and_context_window_backup.json | 10 ++++++++++ model_prices_and_context_window.json | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d3eb93d7267..25affc74f7d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -474,6 +474,16 @@ "mode": "chat", "supports_function_calling": true }, + "azure/command-r-plus": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true + }, "azure/ada": { "max_tokens": 8191, "max_input_tokens": 8191, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d3eb93d7267..25affc74f7d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -474,6 +474,16 @@ "mode": "chat", "supports_function_calling": true }, + "azure/command-r-plus": { + "max_tokens": 4096, + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true + }, "azure/ada": { "max_tokens": 8191, "max_input_tokens": 8191, From 71352b1b36eca94c366fd8ca74bb2ebf8de65c0e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Apr 2024 08:53:24 -0700 Subject: [PATCH 02/10] fix add azure/command-r-plus --- litellm/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/litellm/utils.py b/litellm/utils.py index 17a31751d96..d642fbbb958 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5573,6 +5573,16 @@ def get_llm_provider( dynamic_api_key = None # check if llm provider provided + # AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere + # If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus + if ( + model.split("/", 1)[0] == "azure" + and model.split("/", 1)[1] in litellm.cohere_chat_models + ): + custom_llm_provider = "openai" + model = model.split("/", 1)[1] + return model, custom_llm_provider, dynamic_api_key, api_base + if custom_llm_provider: return model, custom_llm_provider, dynamic_api_key, api_base From 5d196ff300a2ba3d34ff38db53d3150886e99a44 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Apr 2024 08:56:05 -0700 Subject: [PATCH 03/10] test - azure/command-r-plus --- litellm/tests/test_completion.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index b3c0f79f00f..26dd6cc6e9d 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -53,6 +53,25 @@ def test_completion_custom_provider_model_name(): # test_completion_custom_provider_model_name() +def test_completion_azure_command_r(): + try: + litellm.set_verbose = True + + response = completion( + model="azure/command-r-plus", + model_id="Cohere-command-r-plus-gylpd", + api_base=os.getenv("AZURE_COHERE_API_BASE"), + api_key=os.getenv("AZURE_COHERE_API_KEY"), + messages=[{"role": "user", "content": "What is the meaning of life?"}], + ) + + print(response) + except litellm.Timeout as e: + pass + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + def test_completion_claude(): litellm.set_verbose = True litellm.cache = None From b25db0443a4ba4344c640111f996078c6e77b515 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Apr 2024 09:04:38 -0700 Subject: [PATCH 04/10] docs - using command r on azure --- docs/my-website/docs/providers/azure_ai.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/my-website/docs/providers/azure_ai.md b/docs/my-website/docs/providers/azure_ai.md index 4b7a9a783ec..e6ac731d7c5 100644 --- a/docs/my-website/docs/providers/azure_ai.md +++ b/docs/my-website/docs/providers/azure_ai.md @@ -1,5 +1,21 @@ # Azure AI Studio +## Sample Usage +Call Azure Command R Plus in the OpenAI Format +- model=`azure/command-r-plus`. The `azure/` prefix sends this to Azure. The `command-r-plus` indicates the base model being called +- `model_id` = This is your deployment name on Azure AI studio + +```python +import litellm +response = litellm.completion( + model="azure/command-r-plus", + model_id="Cohere-command-r-plus-gylpd", + api_base="https://Cohere-command-r-plus-gylpd-serverless.eastus2.inference.ai.azure.com/v1/" + api_key="eskk******" + messages=[{"role": "user", "content": "What is the meaning of life?"}], +) +``` + ## Using Mistral models deployed on Azure AI Studio ### Sample Usage - setting env vars From 1ba1c84d837f749bce96c96eb143c2932307467e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Apr 2024 09:06:15 -0700 Subject: [PATCH 05/10] skip vertex test for this PR --- litellm/tests/test_custom_callback_input.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/tests/test_custom_callback_input.py b/litellm/tests/test_custom_callback_input.py index 4ee8865b03e..3b3eae1cc24 100644 --- a/litellm/tests/test_custom_callback_input.py +++ b/litellm/tests/test_custom_callback_input.py @@ -652,6 +652,7 @@ def load_vertex_ai_credentials(): @pytest.mark.asyncio +@pytest.mark.skip(reason="Skipping on this PR to test other stuff") async def test_async_chat_vertex_ai_stream(): try: load_vertex_ai_credentials() From cfe358abaab5b6791659b9dd30e0f86b96c25d41 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Apr 2024 09:18:11 -0700 Subject: [PATCH 06/10] simplify calling azure/commmand-r-plus --- docs/my-website/docs/providers/azure_ai.md | 7 ++----- litellm/tests/test_completion.py | 1 - 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/my-website/docs/providers/azure_ai.md b/docs/my-website/docs/providers/azure_ai.md index e6ac731d7c5..2ddb5137bd5 100644 --- a/docs/my-website/docs/providers/azure_ai.md +++ b/docs/my-website/docs/providers/azure_ai.md @@ -1,15 +1,12 @@ # Azure AI Studio ## Sample Usage -Call Azure Command R Plus in the OpenAI Format -- model=`azure/command-r-plus`. The `azure/` prefix sends this to Azure. The `command-r-plus` indicates the base model being called -- `model_id` = This is your deployment name on Azure AI studio +The `azure/` prefix sends this to Azure ```python import litellm response = litellm.completion( - model="azure/command-r-plus", - model_id="Cohere-command-r-plus-gylpd", + model="azure/", api_base="https://Cohere-command-r-plus-gylpd-serverless.eastus2.inference.ai.azure.com/v1/" api_key="eskk******" messages=[{"role": "user", "content": "What is the meaning of life?"}], diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 26dd6cc6e9d..aa54f815496 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -59,7 +59,6 @@ def test_completion_azure_command_r(): response = completion( model="azure/command-r-plus", - model_id="Cohere-command-r-plus-gylpd", api_base=os.getenv("AZURE_COHERE_API_BASE"), api_key=os.getenv("AZURE_COHERE_API_KEY"), messages=[{"role": "user", "content": "What is the meaning of life?"}], From ab60d7c8fb127493456e1ae5eb56c0bdc6c24896 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Apr 2024 09:24:27 -0700 Subject: [PATCH 07/10] docs azure ai command-r plust --- docs/my-website/docs/providers/azure_ai.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/providers/azure_ai.md b/docs/my-website/docs/providers/azure_ai.md index 2ddb5137bd5..fbdc8ee1c98 100644 --- a/docs/my-website/docs/providers/azure_ai.md +++ b/docs/my-website/docs/providers/azure_ai.md @@ -6,8 +6,8 @@ The `azure/` prefix sends this to Azure ```python import litellm response = litellm.completion( - model="azure/", - api_base="https://Cohere-command-r-plus-gylpd-serverless.eastus2.inference.ai.azure.com/v1/" + model="azure/command-r-plus", + api_base="/v1/" api_key="eskk******" messages=[{"role": "user", "content": "What is the meaning of life?"}], ) From 5ce80d82d3d54853ccdb1687f7fc61a8fcb0cbd1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Apr 2024 09:32:39 -0700 Subject: [PATCH 08/10] fix support azure/mistral models --- litellm/__init__.py | 3 +++ litellm/utils.py | 16 +++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index cb07c6a3892..2ed12e7f02e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -260,6 +260,7 @@ open_ai_chat_completion_models: List = [] open_ai_text_completion_models: List = [] cohere_models: List = [] cohere_chat_models: List = [] +mistral_chat_models: List = [] anthropic_models: List = [] openrouter_models: List = [] vertex_language_models: List = [] @@ -284,6 +285,8 @@ for key, value in model_cost.items(): cohere_models.append(key) elif value.get("litellm_provider") == "cohere_chat": cohere_chat_models.append(key) + elif value.get("litellm_provider") == "mistral": + mistral_chat_models.append(key) elif value.get("litellm_provider") == "anthropic": anthropic_models.append(key) elif value.get("litellm_provider") == "openrouter": diff --git a/litellm/utils.py b/litellm/utils.py index d642fbbb958..9da560947af 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5575,13 +5575,15 @@ def get_llm_provider( # AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere # If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus - if ( - model.split("/", 1)[0] == "azure" - and model.split("/", 1)[1] in litellm.cohere_chat_models - ): - custom_llm_provider = "openai" - model = model.split("/", 1)[1] - return model, custom_llm_provider, dynamic_api_key, api_base + if model.split("/", 1)[0] == "azure": + model_name = model.split("/", 1)[1] + if ( + model_name in litellm.cohere_chat_models + or model_name in litellm.mistral_chat_models + ): + custom_llm_provider = "openai" + model = model_name + return model, custom_llm_provider, dynamic_api_key, api_base if custom_llm_provider: return model, custom_llm_provider, dynamic_api_key, api_base From 6b9c04618eb8f1ce42eda61749b47c592041493a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Apr 2024 10:07:43 -0700 Subject: [PATCH 09/10] fix use azure_ai/mistral --- docs/my-website/docs/providers/azure_ai.md | 53 +++++----------------- litellm/utils.py | 2 +- 2 files changed, 12 insertions(+), 43 deletions(-) diff --git a/docs/my-website/docs/providers/azure_ai.md b/docs/my-website/docs/providers/azure_ai.md index fbdc8ee1c98..0166d9848d3 100644 --- a/docs/my-website/docs/providers/azure_ai.md +++ b/docs/my-website/docs/providers/azure_ai.md @@ -13,47 +13,7 @@ response = litellm.completion( ) ``` -## Using Mistral models deployed on Azure AI Studio - -### Sample Usage - setting env vars - -Set `MISTRAL_AZURE_API_KEY` and `MISTRAL_AZURE_API_BASE` in your env - -```shell -MISTRAL_AZURE_API_KEY = "zE************"" -MISTRAL_AZURE_API_BASE = "https://Mistral-large-nmefg-serverless.eastus2.inference.ai.azure.com/v1" -``` - -```python -from litellm import completion -import os - -response = completion( - model="mistral/Mistral-large-dfgfj", - messages=[ - {"role": "user", "content": "hello from litellm"} - ], -) -print(response) -``` - -### Sample Usage - passing `api_base` and `api_key` to `litellm.completion` -```python -from litellm import completion -import os - -response = completion( - model="mistral/Mistral-large-dfgfj", - api_base="https://Mistral-large-dfgfj-serverless.eastus2.inference.ai.azure.com", - api_key = "JGbKodRcTp****" - messages=[ - {"role": "user", "content": "hello from litellm"} - ], -) -print(response) -``` - -### [LiteLLM Proxy] Using Mistral Models +### Sample Usage - LiteLLM Proxy Set this on your litellm proxy config.yaml ```yaml @@ -61,8 +21,17 @@ model_list: - model_name: mistral litellm_params: model: mistral/Mistral-large-dfgfj - api_base: https://Mistral-large-dfgfj-serverless.eastus2.inference.ai.azure.com + api_base: https://Mistral-large-dfgfj-serverless.eastus2.inference.ai.azure.com/v1/ api_key: JGbKodRcTp**** ``` +## Supported Models + +| Model Name | Function Call | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| command-r-plus | `completion(model="azure/command-r-plus", messages)` | +| command-r | `completion(model="azure/command-r", messages)` | +| mistral-large-latest | `completion(model="azure/mistral-large-latest", messages)` | + + diff --git a/litellm/utils.py b/litellm/utils.py index 9da560947af..6c0521265be 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5579,7 +5579,7 @@ def get_llm_provider( model_name = model.split("/", 1)[1] if ( model_name in litellm.cohere_chat_models - or model_name in litellm.mistral_chat_models + or f"mistral/{model_name}" in litellm.mistral_chat_models ): custom_llm_provider = "openai" model = model_name From 22ac95b834745c0e7a6939a2caee1bd4a050e381 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 5 Apr 2024 13:50:56 -0700 Subject: [PATCH 10/10] docs azure_ai command r --- docs/my-website/docs/providers/azure_ai.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/azure_ai.md b/docs/my-website/docs/providers/azure_ai.md index 0166d9848d3..89e57ab5bd3 100644 --- a/docs/my-website/docs/providers/azure_ai.md +++ b/docs/my-website/docs/providers/azure_ai.md @@ -3,6 +3,11 @@ ## Sample Usage The `azure/` prefix sends this to Azure +Ensure you add `/v1` to your api_base. Your Azure AI studio `api_base` passed to litellm should look something like this +```python +api_base = "https://Mistral-large-dfgfj-serverless.eastus2.inference.ai.azure.com/v1/" +``` + ```python import litellm response = litellm.completion( @@ -13,7 +18,7 @@ response = litellm.completion( ) ``` -### Sample Usage - LiteLLM Proxy +## Sample Usage - LiteLLM Proxy Set this on your litellm proxy config.yaml ```yaml