From 602b25972d104e31ac80823dc1a6e50c893b8291 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 23 Feb 2024 10:40:33 -0800 Subject: [PATCH 1/3] (docs) add groq ai --- README.md | 1 + docs/my-website/docs/providers/groq.md | 49 ++++++++++++++++++++++++++ docs/my-website/sidebars.js | 1 + 3 files changed, 51 insertions(+) create mode 100644 docs/my-website/docs/providers/groq.md diff --git a/README.md b/README.md index a09025d4693..6200aa47fc2 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,7 @@ curl 'http://0.0.0.0:8000/key/generate' \ | [ollama](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | | [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | ✅ | | [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | ✅ | +| [Groq AI](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | ✅ | | [anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | ✅ | | [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ | | [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | diff --git a/docs/my-website/docs/providers/groq.md b/docs/my-website/docs/providers/groq.md new file mode 100644 index 00000000000..4a2888312d4 --- /dev/null +++ b/docs/my-website/docs/providers/groq.md @@ -0,0 +1,49 @@ +# Groq +https://groq.com/ + +## API Key +```python +# env variable +os.environ['GROQ_API_KEY'] +``` + +## Sample Usage +```python +from litellm import completion +import os + +os.environ['GROQ_API_KEY'] = "" +response = completion( + model="groq/llama2-70b-4096", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], +) +print(response) +``` + +## Sample Usage - Streaming +```python +from litellm import completion +import os + +os.environ['GROQ_API_KEY'] = "" +response = completion( + model="groq/llama2-70b-4096", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], + stream=True +) + +for chunk in response: + print(chunk) +``` + + +## Supported Models - ALL Groq Models Supported! +We support ALL Groq models, just set `groq/` as a prefix when sending completion requests + +| Model Name | Function Call | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` | diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 27472429133..72c9cf19611 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -131,6 +131,7 @@ const sidebars = { "providers/bedrock", "providers/anyscale", "providers/perplexity", + "providers/groq", "providers/vllm", "providers/xinference", "providers/cloudflare_workers", From e80fcc2762267075f84b197310c5c506e1a3f421 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 23 Feb 2024 10:40:46 -0800 Subject: [PATCH 2/3] (feat) add groq ai --- litellm/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index ac657fa9966..e75f635f306 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -307,6 +307,7 @@ openai_compatible_endpoints: List = [ "api.endpoints.anyscale.com/v1", "api.deepinfra.com/v1/openai", "api.mistral.ai/v1", + "api.groq.com/openai/v1", "api.together.xyz/v1", ] @@ -314,6 +315,7 @@ openai_compatible_endpoints: List = [ openai_compatible_providers: List = [ "anyscale", "mistral", + "groq", "deepinfra", "perplexity", "xinference", @@ -461,6 +463,7 @@ provider_list: List = [ "perplexity", "anyscale", "mistral", + "groq", "maritalk", "voyage", "cloudflare", From 30aa5eaa3405c16b34d2e33dc6a782d096995573 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Fri, 23 Feb 2024 10:42:51 -0800 Subject: [PATCH 3/3] (feat) add groq ai --- litellm/main.py | 6 +++++- litellm/utils.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index acd2ba51358..bb53739dbbb 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -258,6 +258,7 @@ async def acompletion( or custom_llm_provider == "openrouter" or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" + or custom_llm_provider == "groq" or custom_llm_provider == "text-completion-openai" or custom_llm_provider == "huggingface" or custom_llm_provider == "ollama" @@ -810,6 +811,7 @@ def completion( or custom_llm_provider == "custom_openai" or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" + or custom_llm_provider == "groq" or custom_llm_provider == "anyscale" or custom_llm_provider == "mistral" or custom_llm_provider == "openai" @@ -819,7 +821,7 @@ def completion( # note: if a user sets a custom base - we should ensure this works # allow for the setting of dynamic and stateful api-bases api_base = ( - api_base # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api base from there + api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there or litellm.api_base or get_secret("OPENAI_API_BASE") or "https://api.openai.com/v1" @@ -2241,6 +2243,7 @@ async def aembedding(*args, **kwargs): or custom_llm_provider == "openrouter" or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" + or custom_llm_provider == "groq" or custom_llm_provider == "ollama" or custom_llm_provider == "vertex_ai" ): # currently implemented aiohttp calls for just azure and openai, soon all. @@ -2735,6 +2738,7 @@ async def atext_completion(*args, **kwargs): or custom_llm_provider == "openrouter" or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" + or custom_llm_provider == "groq" or custom_llm_provider == "text-completion-openai" or custom_llm_provider == "huggingface" or custom_llm_provider == "ollama" diff --git a/litellm/utils.py b/litellm/utils.py index 8b469294cb5..4468d7c501e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4862,6 +4862,10 @@ def get_llm_provider( # deepinfra is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1 api_base = "https://api.deepinfra.com/v1/openai" dynamic_api_key = get_secret("DEEPINFRA_API_KEY") + elif custom_llm_provider == "groq": + # groq is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.groq.com/openai/v1 + api_base = "https://api.groq.com/openai/v1" + dynamic_api_key = get_secret("GROQ_API_KEY") elif custom_llm_provider == "mistral": # mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai api_base = "https://api.mistral.ai/v1" @@ -4899,6 +4903,9 @@ def get_llm_provider( elif endpoint == "api.mistral.ai/v1": custom_llm_provider = "mistral" dynamic_api_key = get_secret("MISTRAL_API_KEY") + elif endpoint == "api.groq.com/openai/v1": + custom_llm_provider = "groq" + dynamic_api_key = get_secret("GROQ_API_KEY") return model, custom_llm_provider, dynamic_api_key, api_base # check if model in known model provider list -> for huggingface models, raise exception as they don't have a fixed provider (can be togetherai, anyscale, baseten, runpod, et.)