mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
Merge pull request #2168 from BerriAI/litellm_groq_Ai
[FEAT] Support Groq AI
This commit is contained in:
Ishaan Jaff
GitHub
commit
64d268745b
6 changed files with 66 additions and 1 deletions
|
|
@ -209,6 +209,7 @@ curl 'http://0.0.0.0:8000/key/generate' \
|
|||
| [ollama](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ |
|
||||
| [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | ✅ |
|
||||
| [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | ✅ |
|
||||
| [Groq AI](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | ✅ |
|
||||
| [anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | ✅ |
|
||||
| [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ |
|
||||
| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ |
|
||||
|
|
|
|||
49
docs/my-website/docs/providers/groq.md
Normal file
49
docs/my-website/docs/providers/groq.md
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
# Groq
|
||||
https://groq.com/
|
||||
|
||||
## API Key
|
||||
```python
|
||||
# env variable
|
||||
os.environ['GROQ_API_KEY']
|
||||
```
|
||||
|
||||
## Sample Usage
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ['GROQ_API_KEY'] = ""
|
||||
response = completion(
|
||||
model="groq/llama2-70b-4096",
|
||||
messages=[
|
||||
{"role": "user", "content": "hello from litellm"}
|
||||
],
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
## Sample Usage - Streaming
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ['GROQ_API_KEY'] = ""
|
||||
response = completion(
|
||||
model="groq/llama2-70b-4096",
|
||||
messages=[
|
||||
{"role": "user", "content": "hello from litellm"}
|
||||
],
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
```
|
||||
|
||||
|
||||
## Supported Models - ALL Groq Models Supported!
|
||||
We support ALL Groq models, just set `groq/` as a prefix when sending completion requests
|
||||
|
||||
| Model Name | Function Call |
|
||||
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` |
|
||||
|
|
@ -131,6 +131,7 @@ const sidebars = {
|
|||
"providers/bedrock",
|
||||
"providers/anyscale",
|
||||
"providers/perplexity",
|
||||
"providers/groq",
|
||||
"providers/vllm",
|
||||
"providers/xinference",
|
||||
"providers/cloudflare_workers",
|
||||
|
|
|
|||
|
|
@ -307,6 +307,7 @@ openai_compatible_endpoints: List = [
|
|||
"api.endpoints.anyscale.com/v1",
|
||||
"api.deepinfra.com/v1/openai",
|
||||
"api.mistral.ai/v1",
|
||||
"api.groq.com/openai/v1",
|
||||
"api.together.xyz/v1",
|
||||
]
|
||||
|
||||
|
|
@ -314,6 +315,7 @@ openai_compatible_endpoints: List = [
|
|||
openai_compatible_providers: List = [
|
||||
"anyscale",
|
||||
"mistral",
|
||||
"groq",
|
||||
"deepinfra",
|
||||
"perplexity",
|
||||
"xinference",
|
||||
|
|
@ -461,6 +463,7 @@ provider_list: List = [
|
|||
"perplexity",
|
||||
"anyscale",
|
||||
"mistral",
|
||||
"groq",
|
||||
"maritalk",
|
||||
"voyage",
|
||||
"cloudflare",
|
||||
|
|
|
|||
|
|
@ -258,6 +258,7 @@ async def acompletion(
|
|||
or custom_llm_provider == "openrouter"
|
||||
or custom_llm_provider == "deepinfra"
|
||||
or custom_llm_provider == "perplexity"
|
||||
or custom_llm_provider == "groq"
|
||||
or custom_llm_provider == "text-completion-openai"
|
||||
or custom_llm_provider == "huggingface"
|
||||
or custom_llm_provider == "ollama"
|
||||
|
|
@ -810,6 +811,7 @@ def completion(
|
|||
or custom_llm_provider == "custom_openai"
|
||||
or custom_llm_provider == "deepinfra"
|
||||
or custom_llm_provider == "perplexity"
|
||||
or custom_llm_provider == "groq"
|
||||
or custom_llm_provider == "anyscale"
|
||||
or custom_llm_provider == "mistral"
|
||||
or custom_llm_provider == "openai"
|
||||
|
|
@ -819,7 +821,7 @@ def completion(
|
|||
# note: if a user sets a custom base - we should ensure this works
|
||||
# allow for the setting of dynamic and stateful api-bases
|
||||
api_base = (
|
||||
api_base # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api base from there
|
||||
api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
or litellm.api_base
|
||||
or get_secret("OPENAI_API_BASE")
|
||||
or "https://api.openai.com/v1"
|
||||
|
|
@ -2241,6 +2243,7 @@ async def aembedding(*args, **kwargs):
|
|||
or custom_llm_provider == "openrouter"
|
||||
or custom_llm_provider == "deepinfra"
|
||||
or custom_llm_provider == "perplexity"
|
||||
or custom_llm_provider == "groq"
|
||||
or custom_llm_provider == "ollama"
|
||||
or custom_llm_provider == "vertex_ai"
|
||||
): # currently implemented aiohttp calls for just azure and openai, soon all.
|
||||
|
|
@ -2735,6 +2738,7 @@ async def atext_completion(*args, **kwargs):
|
|||
or custom_llm_provider == "openrouter"
|
||||
or custom_llm_provider == "deepinfra"
|
||||
or custom_llm_provider == "perplexity"
|
||||
or custom_llm_provider == "groq"
|
||||
or custom_llm_provider == "text-completion-openai"
|
||||
or custom_llm_provider == "huggingface"
|
||||
or custom_llm_provider == "ollama"
|
||||
|
|
|
|||
|
|
@ -4862,6 +4862,10 @@ def get_llm_provider(
|
|||
# deepinfra is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1
|
||||
api_base = "https://api.deepinfra.com/v1/openai"
|
||||
dynamic_api_key = get_secret("DEEPINFRA_API_KEY")
|
||||
elif custom_llm_provider == "groq":
|
||||
# groq is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.groq.com/openai/v1
|
||||
api_base = "https://api.groq.com/openai/v1"
|
||||
dynamic_api_key = get_secret("GROQ_API_KEY")
|
||||
elif custom_llm_provider == "mistral":
|
||||
# mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai
|
||||
api_base = "https://api.mistral.ai/v1"
|
||||
|
|
@ -4899,6 +4903,9 @@ def get_llm_provider(
|
|||
elif endpoint == "api.mistral.ai/v1":
|
||||
custom_llm_provider = "mistral"
|
||||
dynamic_api_key = get_secret("MISTRAL_API_KEY")
|
||||
elif endpoint == "api.groq.com/openai/v1":
|
||||
custom_llm_provider = "groq"
|
||||
dynamic_api_key = get_secret("GROQ_API_KEY")
|
||||
return model, custom_llm_provider, dynamic_api_key, api_base
|
||||
|
||||
# check if model in known model provider list -> for huggingface models, raise exception as they don't have a fixed provider (can be togetherai, anyscale, baseten, runpod, et.)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue