From be7e6b1c533785890e9d393b0a5c1dd3eefd737b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 1 Aug 2024 16:46:41 -0700 Subject: [PATCH 1/4] add new dbrx models --- ...odel_prices_and_context_window_backup.json | 22 ++++++++++++++++++- model_prices_and_context_window.json | 22 ++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 380771c668a..7cd82494ded 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4698,6 +4698,26 @@ "litellm_provider": "voyage", "mode": "embedding" }, + "databricks/databricks-meta-llama-3-1-405b-instruct": { + "max_tokens": 512, + "max_input_tokens": 512, + "output_vector_size": 1024, + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000015, + "litellm_provider": "databricks", + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/foundation-model-serving" + }, + "databricks/databricks-meta-llama-3-1-70b-instruct": { + "max_tokens": 512, + "max_input_tokens": 512, + "output_vector_size": 1024, + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000003, + "litellm_provider": "databricks", + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/foundation-model-serving" + }, "databricks/databricks-dbrx-instruct": { "max_tokens": 32768, "max_input_tokens": 32768, @@ -4768,5 +4788,5 @@ "litellm_provider": "databricks", "mode": "embedding", "source": "https://www.databricks.com/product/pricing/foundation-model-serving" - } + }, } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 380771c668a..7cd82494ded 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4698,6 +4698,26 @@ "litellm_provider": "voyage", "mode": "embedding" }, + "databricks/databricks-meta-llama-3-1-405b-instruct": { + "max_tokens": 512, + "max_input_tokens": 512, + "output_vector_size": 1024, + "input_cost_per_token": 0.000005, + "output_cost_per_token": 0.000015, + "litellm_provider": "databricks", + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/foundation-model-serving" + }, + "databricks/databricks-meta-llama-3-1-70b-instruct": { + "max_tokens": 512, + "max_input_tokens": 512, + "output_vector_size": 1024, + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000003, + "litellm_provider": "databricks", + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/foundation-model-serving" + }, "databricks/databricks-dbrx-instruct": { "max_tokens": 32768, "max_input_tokens": 32768, @@ -4768,5 +4788,5 @@ "litellm_provider": "databricks", "mode": "embedding", "source": "https://www.databricks.com/product/pricing/foundation-model-serving" - } + }, } From a52ffb93654a22a2b17dac879a31c0eaa1d02071 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 1 Aug 2024 16:50:11 -0700 Subject: [PATCH 2/4] add correct context window --- litellm/model_prices_and_context_window_backup.json | 12 ++++++------ model_prices_and_context_window.json | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7cd82494ded..a510fd26306 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4699,9 +4699,9 @@ "mode": "embedding" }, "databricks/databricks-meta-llama-3-1-405b-instruct": { - "max_tokens": 512, - "max_input_tokens": 512, - "output_vector_size": 1024, + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, "input_cost_per_token": 0.000005, "output_cost_per_token": 0.000015, "litellm_provider": "databricks", @@ -4709,9 +4709,9 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, "databricks/databricks-meta-llama-3-1-70b-instruct": { - "max_tokens": 512, - "max_input_tokens": 512, - "output_vector_size": 1024, + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, "input_cost_per_token": 0.000001, "output_cost_per_token": 0.000003, "litellm_provider": "databricks", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7cd82494ded..a510fd26306 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4699,9 +4699,9 @@ "mode": "embedding" }, "databricks/databricks-meta-llama-3-1-405b-instruct": { - "max_tokens": 512, - "max_input_tokens": 512, - "output_vector_size": 1024, + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, "input_cost_per_token": 0.000005, "output_cost_per_token": 0.000015, "litellm_provider": "databricks", @@ -4709,9 +4709,9 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, "databricks/databricks-meta-llama-3-1-70b-instruct": { - "max_tokens": 512, - "max_input_tokens": 512, - "output_vector_size": 1024, + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, "input_cost_per_token": 0.000001, "output_cost_per_token": 0.000003, "litellm_provider": "databricks", From 8e1a7c0b2de268785bdaa9f0489bdfbe8580fef9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 1 Aug 2024 16:51:28 -0700 Subject: [PATCH 3/4] docs add new dbrx models --- docs/my-website/docs/providers/databricks.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/my-website/docs/providers/databricks.md b/docs/my-website/docs/providers/databricks.md index 633350d2203..c24a6220e2e 100644 --- a/docs/my-website/docs/providers/databricks.md +++ b/docs/my-website/docs/providers/databricks.md @@ -187,6 +187,8 @@ response = litellm.embedding( | Model Name | Command | |----------------------------|------------------------------------------------------------------| +| databricks-meta-llama-3-1-70b-instruct | `completion(model='databricks/databricks-meta-llama-3-1-70b-instruct', messages=messages)` | +| databricks-meta-llama-3-1-405b-instruct | `completion(model='databricks/databricks-meta-llama-3-1-405b-instruct', messages=messages)` | | databricks-dbrx-instruct | `completion(model='databricks/databricks-dbrx-instruct', messages=messages)` | | databricks-meta-llama-3-70b-instruct | `completion(model='databricks/databricks-meta-llama-3-70b-instruct', messages=messages)` | | databricks-llama-2-70b-chat | `completion(model='databricks/databricks-llama-2-70b-chat', messages=messages)` | From 1ce847f7c431412a0a898f61c3edd773d4d13672 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 1 Aug 2024 16:52:52 -0700 Subject: [PATCH 4/4] docs dbrx --- docs/my-website/docs/providers/databricks.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/my-website/docs/providers/databricks.md b/docs/my-website/docs/providers/databricks.md index c24a6220e2e..395a544db45 100644 --- a/docs/my-website/docs/providers/databricks.md +++ b/docs/my-website/docs/providers/databricks.md @@ -5,6 +5,11 @@ import TabItem from '@theme/TabItem'; LiteLLM supports all models on Databricks +:::tip + +**We support ALL Databricks models, just set `model=databricks/` as a prefix when sending litellm requests** + +::: ## Usage @@ -185,6 +190,13 @@ response = litellm.embedding( ## Supported Databricks Chat Completion Models +:::tip + +**We support ALL Databricks models, just set `model=databricks/` as a prefix when sending litellm requests** + +::: + + | Model Name | Command | |----------------------------|------------------------------------------------------------------| | databricks-meta-llama-3-1-70b-instruct | `completion(model='databricks/databricks-meta-llama-3-1-70b-instruct', messages=messages)` | @@ -198,6 +210,13 @@ response = litellm.embedding( ## Supported Databricks Embedding Models +:::tip + +**We support ALL Databricks models, just set `model=databricks/` as a prefix when sending litellm requests** + +::: + + | Model Name | Command | |----------------------------|------------------------------------------------------------------| | databricks-bge-large-en | `embedding(model='databricks/databricks-bge-large-en', messages=messages)` |