mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-08 22:21:35 +00:00
Update ai gateway docs
This commit is contained in:
parent
1ea28ccf67
commit
9783a8aa7c
3 changed files with 187 additions and 16 deletions
|
|
@ -72,6 +72,158 @@ response = completion(
|
|||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="anthropic" label="Anthropic">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
## set ENV variables
|
||||
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
|
||||
|
||||
response = completion(
|
||||
model="anthropic/claude-3-sonnet-20240229",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}]
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="xai" label="xAI">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
## set ENV variables
|
||||
os.environ["XAI_API_KEY"] = "your-api-key"
|
||||
|
||||
response = completion(
|
||||
model="xai/grok-2-latest",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}]
|
||||
)
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="vertex" label="VertexAI">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
# auth: run 'gcloud auth application-default'
|
||||
os.environ["VERTEXAI_PROJECT"] = "hardy-device-386718"
|
||||
os.environ["VERTEXAI_LOCATION"] = "us-central1"
|
||||
|
||||
response = completion(
|
||||
model="vertex_ai/gemini-1.5-pro",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}]
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="nvidia" label="NVIDIA">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
## set ENV variables
|
||||
os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key"
|
||||
os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url"
|
||||
|
||||
response = completion(
|
||||
model="nvidia_nim/<model_name>",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}]
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="hugging" label="HuggingFace">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key"
|
||||
|
||||
# e.g. Call 'WizardLM/WizardCoder-Python-34B-V1.0' hosted on HF Inference endpoints
|
||||
response = completion(
|
||||
model="huggingface/WizardLM/WizardCoder-Python-34B-V1.0",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}],
|
||||
api_base="https://my-endpoint.huggingface.cloud"
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="azure" label="Azure OpenAI">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
## set ENV variables
|
||||
os.environ["AZURE_API_KEY"] = ""
|
||||
os.environ["AZURE_API_BASE"] = ""
|
||||
os.environ["AZURE_API_VERSION"] = ""
|
||||
|
||||
# azure call
|
||||
response = completion(
|
||||
"azure/<your_deployment_name>",
|
||||
messages = [{ "content": "Hello, how are you?","role": "user"}]
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="ollama" label="Ollama">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="ollama/llama2",
|
||||
messages = [{ "content": "Hello, how are you?","role": "user"}],
|
||||
api_base="http://localhost:11434"
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="or" label="Openrouter">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
## set ENV variables
|
||||
os.environ["OPENROUTER_API_KEY"] = "openrouter_api_key"
|
||||
|
||||
response = completion(
|
||||
model="openrouter/google/palm-2-chat-bison",
|
||||
messages = [{ "content": "Hello, how are you?","role": "user"}],
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="novita" label="Novita AI">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key
|
||||
os.environ["NOVITA_API_KEY"] = "novita-api-key"
|
||||
|
||||
response = completion(
|
||||
model="novita/deepseek/deepseek-r1",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}]
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="vercel" label="Vercel AI Gateway">
|
||||
|
|
@ -80,13 +232,12 @@ response = completion(
|
|||
from litellm import completion
|
||||
import os
|
||||
|
||||
## set ENV variables
|
||||
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
|
||||
## set ENV variables.
|
||||
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
|
||||
|
||||
response = completion(
|
||||
model="vercel_ai_gateway/openai/gpt-4o",
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}],
|
||||
stream=True,
|
||||
messages=[{ "content": "Hello, how are you?","role": "user"}]
|
||||
)
|
||||
```
|
||||
|
||||
|
|
@ -312,6 +463,24 @@ response = completion(
|
|||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="vercel" label="Vercel AI Gateway">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
## set ENV variables. V
|
||||
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
|
||||
|
||||
response = completion(
|
||||
model="vercel_ai_gateway/openai/gpt-4o",
|
||||
messages = [{ "content": "Hello, how are you?","role": "user"}],
|
||||
stream=True,
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
</Tabs>
|
||||
|
||||
### Streaming Response Format (OpenAI Format)
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@ os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "" # your Vercel AI Gateway API key
|
|||
os.environ["VERCEL_OIDC_TOKEN"] = "" # your Vercel OIDC token for authentication
|
||||
```
|
||||
|
||||
Note: see the [Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key) for instructions on obtaining a key.
|
||||
|
||||
## Usage - LiteLLM Python SDK
|
||||
|
||||
### Non-streaming
|
||||
|
|
|
|||
|
|
@ -17693,10 +17693,10 @@
|
|||
"mode": "chat"
|
||||
},
|
||||
"vercel_ai_gateway/deepseek/deepseek-r1": {
|
||||
"max_tokens": 64000,
|
||||
"max_tokens": 128000,
|
||||
"input_cost_per_token": 5.5e-07,
|
||||
"output_cost_per_token": 2.19e-06,
|
||||
"max_input_tokens": 64000,
|
||||
"max_input_tokens": 128000,
|
||||
"litellm_provider": "vercel_ai_gateway",
|
||||
"mode": "chat"
|
||||
},
|
||||
|
|
@ -17999,10 +17999,10 @@
|
|||
"mode": "chat"
|
||||
},
|
||||
"vercel_ai_gateway/meta/llama-3.1-8b": {
|
||||
"max_tokens": 128000,
|
||||
"max_tokens": 131000,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 8e-08,
|
||||
"max_input_tokens": 128000,
|
||||
"max_input_tokens": 131000,
|
||||
"litellm_provider": "vercel_ai_gateway",
|
||||
"mode": "chat"
|
||||
},
|
||||
|
|
@ -18426,10 +18426,10 @@
|
|||
"mode": "chat"
|
||||
},
|
||||
"vercel_ai_gateway/openai/gpt-3.5-turbo": {
|
||||
"max_tokens": 4096,
|
||||
"max_tokens": 16385,
|
||||
"input_cost_per_token": 5e-07,
|
||||
"output_cost_per_token": 1.5e-06,
|
||||
"max_input_tokens": 4096,
|
||||
"max_input_tokens": 16385,
|
||||
"litellm_provider": "vercel_ai_gateway",
|
||||
"mode": "chat"
|
||||
},
|
||||
|
|
@ -19278,10 +19278,10 @@
|
|||
"mode": "chat"
|
||||
},
|
||||
"vercel_ai_gateway/mistral/mixtral-8x22b-instruct": {
|
||||
"max_tokens": 2048,
|
||||
"max_tokens": 65536,
|
||||
"input_cost_per_token": 1.2e-06,
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"max_input_tokens": 2048,
|
||||
"max_input_tokens": 65536,
|
||||
"litellm_provider": "vercel_ai_gateway",
|
||||
"mode": "chat"
|
||||
},
|
||||
|
|
@ -19496,9 +19496,9 @@
|
|||
},
|
||||
"openrouter/qwen/qwen3-235b-a22b-2507": {
|
||||
"max_tokens": 262144,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"input_cost_per_token": 1.179e-07,
|
||||
"max_output_tokens": 262144,
|
||||
"output_cost_per_token": 5.9e-07,
|
||||
"output_cost_per_token": 1.179e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat"
|
||||
},
|
||||
|
|
@ -19743,10 +19743,10 @@
|
|||
"mode": "chat"
|
||||
},
|
||||
"vercel_ai_gateway/openai/gpt-3.5-turbo-instruct": {
|
||||
"max_tokens": 4096,
|
||||
"max_tokens": 8192,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"max_input_tokens": 4096,
|
||||
"max_input_tokens": 8192,
|
||||
"litellm_provider": "vercel_ai_gateway",
|
||||
"mode": "chat"
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue