Update ai gateway docs

This commit is contained in:
joshualipman123 2025-07-30 10:17:54 -07:00
parent 1ea28ccf67
commit 9783a8aa7c
3 changed files with 187 additions and 16 deletions

View file

@ -72,6 +72,158 @@ response = completion(
)
```
</TabItem>
<TabItem value="anthropic" label="Anthropic">
```python
from litellm import completion
import os
## set ENV variables
os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
response = completion(
model="anthropic/claude-3-sonnet-20240229",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="xai" label="xAI">
```python
from litellm import completion
import os
## set ENV variables
os.environ["XAI_API_KEY"] = "your-api-key"
response = completion(
model="xai/grok-2-latest",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="vertex" label="VertexAI">
```python
from litellm import completion
import os
# auth: run 'gcloud auth application-default'
os.environ["VERTEXAI_PROJECT"] = "hardy-device-386718"
os.environ["VERTEXAI_LOCATION"] = "us-central1"
response = completion(
model="vertex_ai/gemini-1.5-pro",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="nvidia" label="NVIDIA">
```python
from litellm import completion
import os
## set ENV variables
os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key"
os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url"
response = completion(
model="nvidia_nim/<model_name>",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="hugging" label="HuggingFace">
```python
from litellm import completion
import os
os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key"
# e.g. Call 'WizardLM/WizardCoder-Python-34B-V1.0' hosted on HF Inference endpoints
response = completion(
model="huggingface/WizardLM/WizardCoder-Python-34B-V1.0",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="https://my-endpoint.huggingface.cloud"
)
print(response)
```
</TabItem>
<TabItem value="azure" label="Azure OpenAI">
```python
from litellm import completion
import os
## set ENV variables
os.environ["AZURE_API_KEY"] = ""
os.environ["AZURE_API_BASE"] = ""
os.environ["AZURE_API_VERSION"] = ""
# azure call
response = completion(
"azure/<your_deployment_name>",
messages = [{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="ollama" label="Ollama">
```python
from litellm import completion
response = completion(
model="ollama/llama2",
messages = [{ "content": "Hello, how are you?","role": "user"}],
api_base="http://localhost:11434"
)
```
</TabItem>
<TabItem value="or" label="Openrouter">
```python
from litellm import completion
import os
## set ENV variables
os.environ["OPENROUTER_API_KEY"] = "openrouter_api_key"
response = completion(
model="openrouter/google/palm-2-chat-bison",
messages = [{ "content": "Hello, how are you?","role": "user"}],
)
```
</TabItem>
<TabItem value="novita" label="Novita AI">
```python
from litellm import completion
import os
## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key
os.environ["NOVITA_API_KEY"] = "novita-api-key"
response = completion(
model="novita/deepseek/deepseek-r1",
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="vercel" label="Vercel AI Gateway">
@ -80,13 +232,12 @@ response = completion(
from litellm import completion
import os
## set ENV variables
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-api-key"
## set ENV variables.
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
messages=[{ "content": "Hello, how are you?","role": "user"}],
stream=True,
messages=[{ "content": "Hello, how are you?","role": "user"}]
)
```
@ -312,6 +463,24 @@ response = completion(
</TabItem>
<TabItem value="vercel" label="Vercel AI Gateway">
```python
from litellm import completion
import os
## set ENV variables. V
os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
response = completion(
model="vercel_ai_gateway/openai/gpt-4o",
messages = [{ "content": "Hello, how are you?","role": "user"}],
stream=True,
)
```
</TabItem>
</Tabs>
### Streaming Response Format (OpenAI Format)

View file

@ -28,6 +28,8 @@ os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "" # your Vercel AI Gateway API key
os.environ["VERCEL_OIDC_TOKEN"] = "" # your Vercel OIDC token for authentication
```
Note: see the [Vercel AI Gateway docs](https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key) for instructions on obtaining a key.
## Usage - LiteLLM Python SDK
### Non-streaming

View file

@ -17693,10 +17693,10 @@
"mode": "chat"
},
"vercel_ai_gateway/deepseek/deepseek-r1": {
"max_tokens": 64000,
"max_tokens": 128000,
"input_cost_per_token": 5.5e-07,
"output_cost_per_token": 2.19e-06,
"max_input_tokens": 64000,
"max_input_tokens": 128000,
"litellm_provider": "vercel_ai_gateway",
"mode": "chat"
},
@ -17999,10 +17999,10 @@
"mode": "chat"
},
"vercel_ai_gateway/meta/llama-3.1-8b": {
"max_tokens": 128000,
"max_tokens": 131000,
"input_cost_per_token": 5e-08,
"output_cost_per_token": 8e-08,
"max_input_tokens": 128000,
"max_input_tokens": 131000,
"litellm_provider": "vercel_ai_gateway",
"mode": "chat"
},
@ -18426,10 +18426,10 @@
"mode": "chat"
},
"vercel_ai_gateway/openai/gpt-3.5-turbo": {
"max_tokens": 4096,
"max_tokens": 16385,
"input_cost_per_token": 5e-07,
"output_cost_per_token": 1.5e-06,
"max_input_tokens": 4096,
"max_input_tokens": 16385,
"litellm_provider": "vercel_ai_gateway",
"mode": "chat"
},
@ -19278,10 +19278,10 @@
"mode": "chat"
},
"vercel_ai_gateway/mistral/mixtral-8x22b-instruct": {
"max_tokens": 2048,
"max_tokens": 65536,
"input_cost_per_token": 1.2e-06,
"output_cost_per_token": 1.2e-06,
"max_input_tokens": 2048,
"max_input_tokens": 65536,
"litellm_provider": "vercel_ai_gateway",
"mode": "chat"
},
@ -19496,9 +19496,9 @@
},
"openrouter/qwen/qwen3-235b-a22b-2507": {
"max_tokens": 262144,
"input_cost_per_token": 1.2e-07,
"input_cost_per_token": 1.179e-07,
"max_output_tokens": 262144,
"output_cost_per_token": 5.9e-07,
"output_cost_per_token": 1.179e-07,
"litellm_provider": "openrouter",
"mode": "chat"
},
@ -19743,10 +19743,10 @@
"mode": "chat"
},
"vercel_ai_gateway/openai/gpt-3.5-turbo-instruct": {
"max_tokens": 4096,
"max_tokens": 8192,
"input_cost_per_token": 1.5e-06,
"output_cost_per_token": 2e-06,
"max_input_tokens": 4096,
"max_input_tokens": 8192,
"litellm_provider": "vercel_ai_gateway",
"mode": "chat"
},