mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
Revert "Litellm dev 07 21 2025 p1 (#12848)"
This reverts commit e4e10aa4ed.
This commit is contained in:
Ishaan Jaff
parent
f0a8abb911
commit
bf300f8ca7
59 changed files with 2938 additions and 242 deletions
|
|
@ -18,13 +18,17 @@ model_list:
|
|||
|
||||
### 1 Instance LiteLLM Proxy
|
||||
|
||||
In these tests the median latency of directly calling the fake-openai-endpoint is 60ms.
|
||||
In these tests the baseline latency characteristics are measured against a fake-openai-endpoint.
|
||||
|
||||
| Metric | Litellm Proxy (1 Instance) |
|
||||
|--------|------------------------|
|
||||
| RPS | 475 |
|
||||
| Median Latency (ms) | 100 |
|
||||
| Latency overhead added by LiteLLM Proxy | 40ms |
|
||||
#### Performance Metrics
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| **Requests per Second (RPS)** | 475 |
|
||||
| **End-to-End Latency P50 (ms)** | 100 |
|
||||
| **LiteLLM Overhead P50 (ms)** | 3 |
|
||||
| **LiteLLM Overhead P90 (ms)** | 17 |
|
||||
| **LiteLLM Overhead P99 (ms)** | 31 |
|
||||
|
||||
<!-- <Image img={require('../img/1_instance_proxy.png')} /> -->
|
||||
|
||||
|
|
@ -33,7 +37,8 @@ In these tests the median latency of directly calling the fake-openai-endpoint i
|
|||
<Image img={require('../img/instances_vs_rps.png')} /> -->
|
||||
|
||||
#### Key Findings
|
||||
- Single instance: 475 RPS @ 100ms latency
|
||||
- Single instance: 475 RPS @ 100ms median latency
|
||||
- LiteLLM adds 3ms P50 overhead, 17ms P90 overhead, 31ms P99 overhead
|
||||
- 2 LiteLLM instances: 950 RPS @ 100ms latency
|
||||
- 4 LiteLLM instances: 1900 RPS @ 100ms latency
|
||||
|
||||
|
|
@ -54,6 +59,62 @@ Each machine deploying LiteLLM had the following specs:
|
|||
- 2 CPU
|
||||
- 4GB RAM
|
||||
|
||||
## How to measure LiteLLM Overhead
|
||||
|
||||
All responses from litellm will include the `x-litellm-overhead-duration-ms` header, this is the latency overhead in milliseconds added by LiteLLM Proxy.
|
||||
|
||||
|
||||
If you want to measure this on locust you can use the following code:
|
||||
|
||||
```python showLineNumbers title="Locust Code for measuring LiteLLM Overhead"
|
||||
import os
|
||||
import uuid
|
||||
from locust import HttpUser, task, between, events
|
||||
|
||||
# Custom metric to track LiteLLM overhead duration
|
||||
overhead_durations = []
|
||||
|
||||
@events.request.add_listener
|
||||
def on_request(request_type, name, response_time, response_length, response, context, exception, start_time, url, **kwargs):
|
||||
if response and hasattr(response, 'headers'):
|
||||
overhead_duration = response.headers.get('x-litellm-overhead-duration-ms')
|
||||
if overhead_duration:
|
||||
try:
|
||||
duration_ms = float(overhead_duration)
|
||||
overhead_durations.append(duration_ms)
|
||||
# Report as custom metric
|
||||
events.request.fire(
|
||||
request_type="Custom",
|
||||
name="LiteLLM Overhead Duration (ms)",
|
||||
response_time=duration_ms,
|
||||
response_length=0,
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
class MyUser(HttpUser):
|
||||
wait_time = between(0.5, 1) # Random wait time between requests
|
||||
|
||||
def on_start(self):
|
||||
self.api_key = os.getenv('API_KEY', 'sk-1234567890')
|
||||
self.client.headers.update({'Authorization': f'Bearer {self.api_key}'})
|
||||
|
||||
@task
|
||||
def litellm_completion(self):
|
||||
# no cache hits with this
|
||||
payload = {
|
||||
"model": "db-openai-endpoint",
|
||||
"messages": [{"role": "user", "content": f"{uuid.uuid4()} This is a test there will be no cache hits and we'll fill up the context" * 150}],
|
||||
"user": "my-new-end-user-1"
|
||||
}
|
||||
response = self.client.post("chat/completions", json=payload)
|
||||
|
||||
if response.status_code != 200:
|
||||
# log the errors in error.txt
|
||||
with open("error.txt", "a") as error_log:
|
||||
error_log.write(response.text + "\n")
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Logging Callbacks
|
||||
|
|
|
|||
7
docs/my-website/docs/projects/HolmesGPT.md
Normal file
7
docs/my-website/docs/projects/HolmesGPT.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# HolmesGPT
|
||||
|
||||
[HolmesGPT](https://github.com/robusta-dev/holmesgpt) is an AI-powered observability tool designed to enhance incident response and troubleshooting processes. It's like your 24/7 on-call assistant, helps you solve alerts faster with Automatic Correlations, Investigations, and More.
|
||||
|
||||
LiteLLM helps HolmesGPT integrate with multiple LLM providers or bring their own model and self-host it.
|
||||
|
||||
🔗 Try HolmesGPT → [https://github.com/robusta-dev/holmesgpt](https://github.com/robusta-dev/holmesgpt)
|
||||
|
|
@ -618,23 +618,43 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
|
|||
|
||||
### Azure AD Token Refresh - `DefaultAzureCredential`
|
||||
|
||||
Use this if you want to use Azure `DefaultAzureCredential` for Authentication on your requests
|
||||
Use this if you want to use Azure `DefaultAzureCredential` for Authentication on your requests. `DefaultAzureCredential` automatically discovers and uses available Azure credentials from multiple sources.
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
**Option 1: Explicit DefaultAzureCredential (Recommended)**
|
||||
```python
|
||||
from litellm import completion
|
||||
from azure.identity import DefaultAzureCredential, get_bearer_token_provider
|
||||
|
||||
# DefaultAzureCredential automatically discovers credentials from:
|
||||
# - Environment variables (AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID)
|
||||
# - Managed Identity (AKS, Azure VMs, etc.)
|
||||
# - Azure CLI credentials
|
||||
# - And other Azure identity sources
|
||||
token_provider = get_bearer_token_provider(DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default")
|
||||
|
||||
|
||||
response = completion(
|
||||
model = "azure/<your deployment name>", # model = azure/<your deployment name>
|
||||
api_base = "", # azure api base
|
||||
api_version = "", # azure api version
|
||||
azure_ad_token_provider=token_provider
|
||||
azure_ad_token_provider=token_provider,
|
||||
messages = [{"role": "user", "content": "good morning"}],
|
||||
)
|
||||
```
|
||||
|
||||
**Option 2: LiteLLM Auto-Fallback to DefaultAzureCredential**
|
||||
```python
|
||||
import litellm
|
||||
|
||||
# Enable automatic fallback to DefaultAzureCredential
|
||||
litellm.enable_azure_ad_token_refresh = True
|
||||
|
||||
response = litellm.completion(
|
||||
model = "azure/<your deployment name>",
|
||||
api_base = "",
|
||||
api_version = "",
|
||||
messages = [{"role": "user", "content": "good morning"}],
|
||||
)
|
||||
```
|
||||
|
|
@ -642,6 +662,8 @@ response = completion(
|
|||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY config.yaml">
|
||||
|
||||
**Scenario 1: With Environment Variables (Traditional)**
|
||||
|
||||
1. Add relevant env vars
|
||||
|
||||
```bash
|
||||
|
|
@ -663,12 +685,48 @@ litellm_settings:
|
|||
enable_azure_ad_token_refresh: true # 👈 KEY CHANGE
|
||||
```
|
||||
|
||||
**Scenario 2: Managed Identity (AKS, Azure VMs) - No Hard-coded Credentials Required**
|
||||
|
||||
Perfect for AKS clusters, Azure VMs, or other managed environments where Azure automatically injects credentials.
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: azure/your-deployment-name
|
||||
api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
|
||||
|
||||
litellm_settings:
|
||||
enable_azure_ad_token_refresh: true # 👈 KEY CHANGE
|
||||
```
|
||||
|
||||
**Scenario 3: Azure CLI Authentication**
|
||||
|
||||
If you're authenticated via `az login`, no additional configuration needed:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: azure/your-deployment-name
|
||||
api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
|
||||
|
||||
litellm_settings:
|
||||
enable_azure_ad_token_refresh: true # 👈 KEY CHANGE
|
||||
```
|
||||
|
||||
3. Start proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
**How it works**:
|
||||
- LiteLLM first tries Service Principal authentication (if environment variables are available)
|
||||
- If that fails, it automatically falls back to `DefaultAzureCredential`
|
||||
- `DefaultAzureCredential` will use Managed Identity, Azure CLI credentials, or other available Azure identity sources
|
||||
- This eliminates the need for hard-coded credentials in managed environments like AKS
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion
|
|||
| mixtral-8x7b-32768 | `completion(model="groq/mixtral-8x7b-32768", messages)` |
|
||||
| gemma-7b-it | `completion(model="groq/gemma-7b-it", messages)` |
|
||||
| moonshotai/kimi-k2-instruct | `completion(model="groq/moonshotai/kimi-k2-instruct", messages)` |
|
||||
| qwen-qwq-32b | `completion(model="groq/qwen-qwq-32b", messages)` |
|
||||
| qwen3-32b | `completion(model="groq/qwen/qwen3-32b", messages)` |
|
||||
|
||||
## Groq - Tool / Function Calling Example
|
||||
|
||||
|
|
|
|||
331
docs/my-website/docs/providers/hyperbolic.md
Normal file
331
docs/my-website/docs/providers/hyperbolic.md
Normal file
|
|
@ -0,0 +1,331 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Hyperbolic
|
||||
|
||||
## Overview
|
||||
|
||||
| Property | Details |
|
||||
|-------|-------|
|
||||
| Description | Hyperbolic provides access to the latest models at a fraction of legacy cloud costs, with OpenAI-compatible APIs for LLMs, image generation, and more. |
|
||||
| Provider Route on LiteLLM | `hyperbolic/` |
|
||||
| Link to Provider Doc | [Hyperbolic Documentation ↗](https://docs.hyperbolic.xyz) |
|
||||
| Base URL | `https://api.hyperbolic.xyz/v1` |
|
||||
| Supported Operations | [`/chat/completions`](#sample-usage) |
|
||||
|
||||
<br />
|
||||
<br />
|
||||
|
||||
https://docs.hyperbolic.xyz
|
||||
|
||||
**We support ALL Hyperbolic models, just set `hyperbolic/` as a prefix when sending completion requests**
|
||||
|
||||
## Available Models
|
||||
|
||||
### Language Models
|
||||
|
||||
| Model | Description | Context Window | Pricing per 1M tokens |
|
||||
|-------|-------------|----------------|----------------------|
|
||||
| `hyperbolic/deepseek-ai/DeepSeek-V3` | DeepSeek V3 - Fast and efficient | 131,072 tokens | $0.25 |
|
||||
| `hyperbolic/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 March 2024 version | 131,072 tokens | $0.25 |
|
||||
| `hyperbolic/deepseek-ai/DeepSeek-R1` | DeepSeek R1 - Reasoning model | 131,072 tokens | $2.00 |
|
||||
| `hyperbolic/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 May 2028 version | 131,072 tokens | $0.25 |
|
||||
| `hyperbolic/Qwen/Qwen2.5-72B-Instruct` | Qwen 2.5 72B Instruct | 131,072 tokens | $0.40 |
|
||||
| `hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct` | Qwen 2.5 Coder 32B for code generation | 131,072 tokens | $0.20 |
|
||||
| `hyperbolic/Qwen/Qwen3-235B-A22B` | Qwen 3 235B A22B variant | 131,072 tokens | $2.00 |
|
||||
| `hyperbolic/Qwen/QwQ-32B` | Qwen QwQ 32B | 131,072 tokens | $0.20 |
|
||||
| `hyperbolic/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B Instruct | 131,072 tokens | $0.80 |
|
||||
| `hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct` | Llama 3.1 405B Instruct | 131,072 tokens | $5.00 |
|
||||
| `hyperbolic/moonshotai/Kimi-K2-Instruct` | Kimi K2 Instruct | 131,072 tokens | $2.00 |
|
||||
|
||||
## Required Variables
|
||||
|
||||
```python showLineNumbers title="Environment Variables"
|
||||
os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key
|
||||
```
|
||||
|
||||
Get your API key from [Hyperbolic dashboard](https://app.hyperbolic.ai).
|
||||
|
||||
## Usage - LiteLLM Python SDK
|
||||
|
||||
### Non-streaming
|
||||
|
||||
```python showLineNumbers title="Hyperbolic Non-streaming Completion"
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key
|
||||
|
||||
messages = [{"content": "What is the capital of France?", "role": "user"}]
|
||||
|
||||
# Hyperbolic call
|
||||
response = completion(
|
||||
model="hyperbolic/Qwen/Qwen2.5-72B-Instruct",
|
||||
messages=messages
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
### Streaming
|
||||
|
||||
```python showLineNumbers title="Hyperbolic Streaming Completion"
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key
|
||||
|
||||
messages = [{"content": "Write a short poem about AI", "role": "user"}]
|
||||
|
||||
# Hyperbolic call with streaming
|
||||
response = completion(
|
||||
model="hyperbolic/deepseek-ai/DeepSeek-V3",
|
||||
messages=messages,
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
```
|
||||
|
||||
### Function Calling
|
||||
|
||||
```python showLineNumbers title="Hyperbolic Function Calling"
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
os.environ["HYPERBOLIC_API_KEY"] = "" # your Hyperbolic API key
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather in a location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA"
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"enum": ["celsius", "fahrenheit"]
|
||||
}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
response = completion(
|
||||
model="hyperbolic/deepseek-ai/DeepSeek-V3",
|
||||
messages=[{"role": "user", "content": "What's the weather like in New York?"}],
|
||||
tools=tools,
|
||||
tool_choice="auto"
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
## Usage - LiteLLM Proxy
|
||||
|
||||
Add the following to your LiteLLM Proxy configuration file:
|
||||
|
||||
```yaml showLineNumbers title="config.yaml"
|
||||
model_list:
|
||||
- model_name: deepseek-fast
|
||||
litellm_params:
|
||||
model: hyperbolic/deepseek-ai/DeepSeek-V3
|
||||
api_key: os.environ/HYPERBOLIC_API_KEY
|
||||
|
||||
- model_name: qwen-coder
|
||||
litellm_params:
|
||||
model: hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct
|
||||
api_key: os.environ/HYPERBOLIC_API_KEY
|
||||
|
||||
- model_name: deepseek-reasoning
|
||||
litellm_params:
|
||||
model: hyperbolic/deepseek-ai/DeepSeek-R1
|
||||
api_key: os.environ/HYPERBOLIC_API_KEY
|
||||
```
|
||||
|
||||
Start your LiteLLM Proxy server:
|
||||
|
||||
```bash showLineNumbers title="Start LiteLLM Proxy"
|
||||
litellm --config config.yaml
|
||||
|
||||
# RUNNING on http://0.0.0.0:4000
|
||||
```
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="openai-sdk" label="OpenAI SDK">
|
||||
|
||||
```python showLineNumbers title="Hyperbolic via Proxy - Non-streaming"
|
||||
from openai import OpenAI
|
||||
|
||||
# Initialize client with your proxy URL
|
||||
client = OpenAI(
|
||||
base_url="http://localhost:4000", # Your proxy URL
|
||||
api_key="your-proxy-api-key" # Your proxy API key
|
||||
)
|
||||
|
||||
# Non-streaming response
|
||||
response = client.chat.completions.create(
|
||||
model="deepseek-fast",
|
||||
messages=[{"role": "user", "content": "Explain quantum computing in simple terms"}]
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
```python showLineNumbers title="Hyperbolic via Proxy - Streaming"
|
||||
from openai import OpenAI
|
||||
|
||||
# Initialize client with your proxy URL
|
||||
client = OpenAI(
|
||||
base_url="http://localhost:4000", # Your proxy URL
|
||||
api_key="your-proxy-api-key" # Your proxy API key
|
||||
)
|
||||
|
||||
# Streaming response
|
||||
response = client.chat.completions.create(
|
||||
model="qwen-coder",
|
||||
messages=[{"role": "user", "content": "Write a Python function to sort a list"}],
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
if chunk.choices[0].delta.content is not None:
|
||||
print(chunk.choices[0].delta.content, end="")
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="litellm-sdk" label="LiteLLM SDK">
|
||||
|
||||
```python showLineNumbers title="Hyperbolic via Proxy - LiteLLM SDK"
|
||||
import litellm
|
||||
|
||||
# Configure LiteLLM to use your proxy
|
||||
response = litellm.completion(
|
||||
model="litellm_proxy/deepseek-fast",
|
||||
messages=[{"role": "user", "content": "What are the benefits of renewable energy?"}],
|
||||
api_base="http://localhost:4000",
|
||||
api_key="your-proxy-api-key"
|
||||
)
|
||||
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
```python showLineNumbers title="Hyperbolic via Proxy - LiteLLM SDK Streaming"
|
||||
import litellm
|
||||
|
||||
# Configure LiteLLM to use your proxy with streaming
|
||||
response = litellm.completion(
|
||||
model="litellm_proxy/qwen-coder",
|
||||
messages=[{"role": "user", "content": "Implement a binary search algorithm"}],
|
||||
api_base="http://localhost:4000",
|
||||
api_key="your-proxy-api-key",
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
|
||||
print(chunk.choices[0].delta.content, end="")
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="curl" label="cURL">
|
||||
|
||||
```bash showLineNumbers title="Hyperbolic via Proxy - cURL"
|
||||
curl http://localhost:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer your-proxy-api-key" \
|
||||
-d '{
|
||||
"model": "deepseek-fast",
|
||||
"messages": [{"role": "user", "content": "What is machine learning?"}]
|
||||
}'
|
||||
```
|
||||
|
||||
```bash showLineNumbers title="Hyperbolic via Proxy - cURL Streaming"
|
||||
curl http://localhost:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer your-proxy-api-key" \
|
||||
-d '{
|
||||
"model": "qwen-coder",
|
||||
"messages": [{"role": "user", "content": "Write a REST API in Python"}],
|
||||
"stream": true
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
|
||||
|
||||
## Supported OpenAI Parameters
|
||||
|
||||
Hyperbolic supports the following OpenAI-compatible parameters:
|
||||
|
||||
| Parameter | Type | Description |
|
||||
|-----------|------|-------------|
|
||||
| `messages` | array | **Required**. Array of message objects with 'role' and 'content' |
|
||||
| `model` | string | **Required**. Model ID (e.g., deepseek-ai/DeepSeek-V3, Qwen/Qwen2.5-72B-Instruct) |
|
||||
| `stream` | boolean | Optional. Enable streaming responses |
|
||||
| `temperature` | float | Optional. Sampling temperature (0.0 to 2.0) |
|
||||
| `top_p` | float | Optional. Nucleus sampling parameter |
|
||||
| `max_tokens` | integer | Optional. Maximum tokens to generate |
|
||||
| `frequency_penalty` | float | Optional. Penalize frequent tokens |
|
||||
| `presence_penalty` | float | Optional. Penalize tokens based on presence |
|
||||
| `stop` | string/array | Optional. Stop sequences |
|
||||
| `n` | integer | Optional. Number of completions to generate |
|
||||
| `tools` | array | Optional. List of available tools/functions |
|
||||
| `tool_choice` | string/object | Optional. Control tool/function calling |
|
||||
| `response_format` | object | Optional. Response format specification |
|
||||
| `seed` | integer | Optional. Random seed for reproducibility |
|
||||
| `user` | string | Optional. User identifier |
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Custom API Base
|
||||
|
||||
If you're using a custom Hyperbolic deployment:
|
||||
|
||||
```python showLineNumbers title="Custom API Base"
|
||||
import litellm
|
||||
|
||||
response = litellm.completion(
|
||||
model="hyperbolic/deepseek-ai/DeepSeek-V3",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
api_base="https://your-custom-hyperbolic-endpoint.com/v1",
|
||||
api_key="your-api-key"
|
||||
)
|
||||
```
|
||||
|
||||
### Rate Limits
|
||||
|
||||
Hyperbolic offers different tiers:
|
||||
- **Basic**: 60 requests per minute (RPM)
|
||||
- **Pro**: 600 RPM
|
||||
- **Enterprise**: Custom limits
|
||||
|
||||
## Pricing
|
||||
|
||||
Hyperbolic offers competitive pay-as-you-go pricing with no hidden fees or long-term commitments. See the model table above for specific pricing per million tokens.
|
||||
|
||||
### Precision Options
|
||||
- **BF16**: Best precision and performance, suitable for tasks where accuracy is critical
|
||||
- **FP8**: Optimized for efficiency and speed, ideal for high-throughput applications at lower cost
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [Hyperbolic Official Documentation](https://docs.hyperbolic.xyz)
|
||||
- [Hyperbolic Dashboard](https://app.hyperbolic.ai)
|
||||
- [API Reference](https://docs.hyperbolic.xyz/docs/rest-api)
|
||||
280
docs/my-website/docs/providers/lambda_ai.md
Normal file
280
docs/my-website/docs/providers/lambda_ai.md
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Lambda AI
|
||||
|
||||
## Overview
|
||||
|
||||
| Property | Details |
|
||||
|-------|-------|
|
||||
| Description | Lambda AI provides access to a wide range of open-source language models through their cloud GPU infrastructure, optimized for inference at scale. |
|
||||
| Provider Route on LiteLLM | `lambda_ai/` |
|
||||
| Link to Provider Doc | [Lambda AI API Documentation ↗](https://docs.lambda.ai/api) |
|
||||
| Base URL | `https://api.lambda.ai/v1` |
|
||||
| Supported Operations | [`/chat/completions`](#sample-usage) |
|
||||
|
||||
<br />
|
||||
<br />
|
||||
|
||||
https://docs.lambda.ai/api
|
||||
|
||||
**We support ALL Lambda AI models, just set `lambda_ai/` as a prefix when sending completion requests**
|
||||
|
||||
## Available Models
|
||||
|
||||
Lambda AI offers a diverse selection of state-of-the-art open-source models:
|
||||
|
||||
### Large Language Models
|
||||
|
||||
| Model | Description | Context Window |
|
||||
|-------|-------------|----------------|
|
||||
| `lambda_ai/llama3.3-70b-instruct-fp8` | Llama 3.3 70B with FP8 quantization | 8,192 tokens |
|
||||
| `lambda_ai/llama3.1-405b-instruct-fp8` | Llama 3.1 405B with FP8 quantization | 8,192 tokens |
|
||||
| `lambda_ai/llama3.1-70b-instruct-fp8` | Llama 3.1 70B with FP8 quantization | 8,192 tokens |
|
||||
| `lambda_ai/llama3.1-8b-instruct` | Llama 3.1 8B instruction-tuned | 8,192 tokens |
|
||||
| `lambda_ai/llama3.1-nemotron-70b-instruct-fp8` | Llama 3.1 Nemotron 70B | 8,192 tokens |
|
||||
|
||||
### DeepSeek Models
|
||||
|
||||
| Model | Description | Context Window |
|
||||
|-------|-------------|----------------|
|
||||
| `lambda_ai/deepseek-llama3.3-70b` | DeepSeek Llama 3.3 70B | 8,192 tokens |
|
||||
| `lambda_ai/deepseek-r1-0528` | DeepSeek R1 0528 | 8,192 tokens |
|
||||
| `lambda_ai/deepseek-r1-671b` | DeepSeek R1 671B | 8,192 tokens |
|
||||
| `lambda_ai/deepseek-v3-0324` | DeepSeek V3 0324 | 8,192 tokens |
|
||||
|
||||
### Hermes Models
|
||||
|
||||
| Model | Description | Context Window |
|
||||
|-------|-------------|----------------|
|
||||
| `lambda_ai/hermes3-405b` | Hermes 3 405B | 8,192 tokens |
|
||||
| `lambda_ai/hermes3-70b` | Hermes 3 70B | 8,192 tokens |
|
||||
| `lambda_ai/hermes3-8b` | Hermes 3 8B | 8,192 tokens |
|
||||
|
||||
### Coding Models
|
||||
|
||||
| Model | Description | Context Window |
|
||||
|-------|-------------|----------------|
|
||||
| `lambda_ai/qwen25-coder-32b-instruct` | Qwen 2.5 Coder 32B | 8,192 tokens |
|
||||
| `lambda_ai/qwen3-32b-fp8` | Qwen 3 32B with FP8 | 8,192 tokens |
|
||||
|
||||
### Vision Models
|
||||
|
||||
| Model | Description | Context Window |
|
||||
|-------|-------------|----------------|
|
||||
| `lambda_ai/llama3.2-11b-vision-instruct` | Llama 3.2 11B with vision capabilities | 8,192 tokens |
|
||||
|
||||
### Specialized Models
|
||||
|
||||
| Model | Description | Context Window |
|
||||
|-------|-------------|----------------|
|
||||
| `lambda_ai/llama-4-maverick-17b-128e-instruct-fp8` | Llama 4 Maverick with 128k context | 131,072 tokens |
|
||||
| `lambda_ai/llama-4-scout-17b-16e-instruct` | Llama 4 Scout with 16k context | 16,384 tokens |
|
||||
| `lambda_ai/lfm-40b` | LFM 40B model | 8,192 tokens |
|
||||
| `lambda_ai/lfm-7b` | LFM 7B model | 8,192 tokens |
|
||||
|
||||
## Required Variables
|
||||
|
||||
```python showLineNumbers title="Environment Variables"
|
||||
os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
|
||||
```
|
||||
|
||||
## Usage - LiteLLM Python SDK
|
||||
|
||||
### Non-streaming
|
||||
|
||||
```python showLineNumbers title="Lambda AI Non-streaming Completion"
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
|
||||
|
||||
messages = [{"content": "Hello, how are you?", "role": "user"}]
|
||||
|
||||
# Lambda AI call
|
||||
response = completion(
|
||||
model="lambda_ai/llama3.1-8b-instruct",
|
||||
messages=messages
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
### Streaming
|
||||
|
||||
```python showLineNumbers title="Lambda AI Streaming Completion"
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
|
||||
|
||||
messages = [{"content": "Write a short story about AI", "role": "user"}]
|
||||
|
||||
# Lambda AI call with streaming
|
||||
response = completion(
|
||||
model="lambda_ai/llama3.1-70b-instruct-fp8",
|
||||
messages=messages,
|
||||
stream=True
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
```
|
||||
|
||||
### Vision/Multimodal Support
|
||||
|
||||
The Llama 3.2 Vision model supports image inputs:
|
||||
|
||||
```python showLineNumbers title="Lambda AI Vision/Multimodal"
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
|
||||
|
||||
messages = [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What's in this image?"
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://example.com/image.jpg"
|
||||
}
|
||||
}
|
||||
]
|
||||
}]
|
||||
|
||||
# Lambda AI vision model call
|
||||
response = completion(
|
||||
model="lambda_ai/llama3.2-11b-vision-instruct",
|
||||
messages=messages
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
### Function Calling
|
||||
|
||||
Lambda AI models support function calling:
|
||||
|
||||
```python showLineNumbers title="Lambda AI Function Calling"
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
os.environ["LAMBDA_API_KEY"] = "" # your Lambda AI API key
|
||||
|
||||
# Define tools
|
||||
tools = [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather in a location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {
|
||||
"type": "string",
|
||||
"description": "The city and state, e.g. San Francisco, CA"
|
||||
}
|
||||
},
|
||||
"required": ["location"]
|
||||
}
|
||||
}
|
||||
}]
|
||||
|
||||
messages = [{"role": "user", "content": "What's the weather in Boston?"}]
|
||||
|
||||
# Lambda AI call with function calling
|
||||
response = completion(
|
||||
model="lambda_ai/hermes3-70b",
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
tool_choice="auto"
|
||||
)
|
||||
|
||||
print(response)
|
||||
```
|
||||
|
||||
## Usage - LiteLLM Proxy Server
|
||||
|
||||
```yaml showLineNumbers title="config.yaml"
|
||||
model_list:
|
||||
- model_name: llama-8b
|
||||
litellm_params:
|
||||
model: lambda_ai/llama3.1-8b-instruct
|
||||
api_key: os.environ/LAMBDA_API_KEY
|
||||
- model_name: deepseek-70b
|
||||
litellm_params:
|
||||
model: lambda_ai/deepseek-llama3.3-70b
|
||||
api_key: os.environ/LAMBDA_API_KEY
|
||||
- model_name: hermes-405b
|
||||
litellm_params:
|
||||
model: lambda_ai/hermes3-405b
|
||||
api_key: os.environ/LAMBDA_API_KEY
|
||||
- model_name: qwen-coder
|
||||
litellm_params:
|
||||
model: lambda_ai/qwen25-coder-32b-instruct
|
||||
api_key: os.environ/LAMBDA_API_KEY
|
||||
```
|
||||
|
||||
## Custom API Base
|
||||
|
||||
If you need to use a custom API base URL:
|
||||
|
||||
```python showLineNumbers title="Custom API Base"
|
||||
import os
|
||||
import litellm
|
||||
from litellm import completion
|
||||
|
||||
# Using environment variable
|
||||
os.environ["LAMBDA_API_BASE"] = "https://custom.lambda-api.com/v1"
|
||||
os.environ["LAMBDA_API_KEY"] = "" # your API key
|
||||
|
||||
# Or pass directly
|
||||
response = completion(
|
||||
model="lambda_ai/llama3.1-8b-instruct",
|
||||
messages=[{"content": "Hello!", "role": "user"}],
|
||||
api_base="https://custom.lambda-api.com/v1",
|
||||
api_key="your-api-key"
|
||||
)
|
||||
```
|
||||
|
||||
## Supported OpenAI Parameters
|
||||
|
||||
Lambda AI supports all standard OpenAI parameters since it's fully OpenAI-compatible:
|
||||
|
||||
- `temperature`
|
||||
- `max_tokens`
|
||||
- `top_p`
|
||||
- `frequency_penalty`
|
||||
- `presence_penalty`
|
||||
- `stop`
|
||||
- `n`
|
||||
- `stream`
|
||||
- `tools`
|
||||
- `tool_choice`
|
||||
- `response_format`
|
||||
- `seed`
|
||||
- `user`
|
||||
- `logit_bias`
|
||||
|
||||
Example with parameters:
|
||||
|
||||
```python showLineNumbers title="Lambda AI with Parameters"
|
||||
response = completion(
|
||||
model="lambda_ai/hermes3-405b",
|
||||
messages=[{"content": "Explain quantum computing", "role": "user"}],
|
||||
temperature=0.7,
|
||||
max_tokens=500,
|
||||
top_p=0.9,
|
||||
frequency_penalty=0.2,
|
||||
presence_penalty=0.1
|
||||
)
|
||||
```
|
||||
|
|
@ -10,7 +10,7 @@ import TabItem from '@theme/TabItem';
|
|||
| Description | Moonshot AI provides large language models including the moonshot-v1 series and kimi models. |
|
||||
| Provider Route on LiteLLM | `moonshot/` |
|
||||
| Link to Provider Doc | [Moonshot AI ↗](https://platform.moonshot.ai/) |
|
||||
| Base URL | `https://api.moonshot.cn/` |
|
||||
| Base URL | `https://api.moonshot.ai/` |
|
||||
| Supported Operations | [`/chat/completions`](#sample-usage) |
|
||||
|
||||
<br />
|
||||
|
|
@ -26,6 +26,18 @@ https://platform.moonshot.ai/
|
|||
os.environ["MOONSHOT_API_KEY"] = "" # your Moonshot AI API key
|
||||
```
|
||||
|
||||
**ATTENTION:**
|
||||
|
||||
Moonshot AI offers two distinct API endpoints: a global one and a China-specific one.
|
||||
- Global API Base URL: `https://api.moonshot.ai/v1` (This is the one currently implemented)
|
||||
- China API Base URL: `https://api.moonshot.cn/v1`
|
||||
|
||||
You can overwrite the base url with:
|
||||
|
||||
```
|
||||
os.environ["MOONSHOT_API_BASE"] = "https://api.moonshot.cn/v1"
|
||||
```
|
||||
|
||||
## Usage - LiteLLM Python SDK
|
||||
|
||||
### Non-streaming
|
||||
|
|
|
|||
123
docs/my-website/docs/providers/morph.md
Normal file
123
docs/my-website/docs/providers/morph.md
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
# Morph
|
||||
|
||||
LiteLLM supports all models on [Morph](https://morphllm.com)
|
||||
|
||||
## Overview
|
||||
|
||||
Morph provides specialized AI models designed for agentic workflows, particularly excelling at precise code editing and manipulation. Their "Apply" models enable targeted code changes without full file rewrites, making them ideal for AI agents that need to make intelligent, context-aware code modifications.
|
||||
|
||||
## API Key
|
||||
```python
|
||||
import os
|
||||
os.environ["MORPH_API_KEY"] = "your-api-key"
|
||||
```
|
||||
|
||||
## Sample Usage
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
# set env variable
|
||||
os.environ["MORPH_API_KEY"] = "your-api-key"
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "Write a Python function to calculate factorial"}
|
||||
]
|
||||
|
||||
## Morph v3 Fast - Optimized for speed
|
||||
response = completion(
|
||||
model="morph/morph-v3-fast",
|
||||
messages=messages,
|
||||
)
|
||||
print(response)
|
||||
|
||||
## Morph v3 Large - Most capable model
|
||||
response = completion(
|
||||
model="morph/morph-v3-large",
|
||||
messages=messages,
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
## Sample Usage - Streaming
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
# set env variable
|
||||
os.environ["MORPH_API_KEY"] = "your-api-key"
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "Write a Python function to calculate factorial"}
|
||||
]
|
||||
|
||||
## Morph v3 Fast with streaming
|
||||
response = completion(
|
||||
model="morph/morph-v3-fast",
|
||||
messages=messages,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
```
|
||||
|
||||
## Supported Models
|
||||
|
||||
| Model Name | Function Call | Description | Context Window |
|
||||
|--------------------------|--------------------------------------------|-----------------------|----------------|
|
||||
| morph-v3-fast | `completion('morph/morph-v3-fast', messages)` | Fastest model, optimized for quick responses | 16k tokens |
|
||||
| morph-v3-large | `completion('morph/morph-v3-large', messages)` | Most capable model for complex tasks | 16k tokens |
|
||||
|
||||
## Usage - LiteLLM Proxy Server
|
||||
|
||||
Here's how to use Morph with the LiteLLM Proxy Server:
|
||||
|
||||
1. Save API key in your environment
|
||||
```bash
|
||||
export MORPH_API_KEY="your-api-key"
|
||||
```
|
||||
|
||||
2. Add model to config.yaml
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: morph-v3-fast
|
||||
litellm_params:
|
||||
model: morph/morph-v3-fast
|
||||
|
||||
- model_name: morph-v3-large
|
||||
litellm_params:
|
||||
model: morph/morph-v3-large
|
||||
```
|
||||
|
||||
3. Start the proxy server
|
||||
```bash
|
||||
litellm --config config.yaml
|
||||
```
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Setting API Base
|
||||
```python
|
||||
import litellm
|
||||
|
||||
# set custom api base
|
||||
response = completion(
|
||||
model="morph/morph-v3-large",
|
||||
messages=[{"role": "user", "content": "Hello, world!"}],
|
||||
api_base="https://api.morphllm.com/v1"
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
||||
### Setting API Key
|
||||
```python
|
||||
import litellm
|
||||
|
||||
# set api key via completion
|
||||
response = completion(
|
||||
model="morph/morph-v3-large",
|
||||
messages=[{"role": "user", "content": "Hello, world!"}],
|
||||
api_key="your-api-key"
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
|
|
@ -313,11 +313,22 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
|
|||
## VertexAI Mistral API
|
||||
|
||||
[**Supported OpenAI Params**](https://github.com/BerriAI/litellm/blob/e0f3cd580cb85066f7d36241a03c30aa50a8a31d/litellm/llms/openai.py#L137)
|
||||
|
||||
**LiteLLM Supports all Vertex AI Mistral Models.** Ensure you use the `vertex_ai/mistral-` prefix for all Vertex AI Mistral models.
|
||||
|
||||
Overview
|
||||
|
||||
| Property | Details |
|
||||
|----------|---------|
|
||||
| Provider Route | `vertex_ai/mistral-{MODEL}` |
|
||||
| Vertex Documentation | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) |
|
||||
|
||||
| Model Name | Function Call |
|
||||
|------------------|--------------------------------------|
|
||||
| mistral-large@latest | `completion('vertex_ai/mistral-large@latest', messages)` |
|
||||
| mistral-large@2407 | `completion('vertex_ai/mistral-large@2407', messages)` |
|
||||
| mistral-small-2503 | `completion('vertex_ai/mistral-small-2503', messages)` |
|
||||
| mistral-large-2411 | `completion('vertex_ai/mistral-large-2411', messages)` |
|
||||
| mistral-nemo@latest | `completion('vertex_ai/mistral-nemo@latest', messages)` |
|
||||
| codestral@latest | `completion('vertex_ai/codestral@latest', messages)` |
|
||||
| codestral@@2405 | `completion('vertex_ai/codestral@2405', messages)` |
|
||||
|
|
|
|||
|
|
@ -189,6 +189,7 @@ general_settings:
|
|||
| proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** |
|
||||
| proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** |
|
||||
| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** |
|
||||
| proxy_batch_polling_interval | int | Time (in seconds) to wait before polling a batch, to check if it's completed. **Default is 6000 seconds (1 hour)** |
|
||||
| alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) |
|
||||
| custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) |
|
||||
| allowed_ips | List[str] | List of IPs allowed to access the proxy. If not set, all IPs are allowed. |
|
||||
|
|
@ -622,6 +623,7 @@ router_settings:
|
|||
| PROXY_ADMIN_ID | Admin identifier for proxy server
|
||||
| PROXY_BASE_URL | Base URL for proxy service
|
||||
| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10
|
||||
| PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour)
|
||||
| PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605
|
||||
| PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597
|
||||
| PROXY_LOGOUT_URL | URL for logging out of the proxy service
|
||||
|
|
|
|||
93
docs/my-website/docs/proxy/guardrails/model_armor.md
Normal file
93
docs/my-website/docs/proxy/guardrails/model_armor.md
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Google Cloud Model Armor
|
||||
|
||||
LiteLLM supports Google Cloud Model Armor guardrails via the [Model Armor API](https://cloud.google.com/security-command-center/docs/model-armor-overview).
|
||||
|
||||
|
||||
## Supported Guardrails
|
||||
|
||||
- [Model Armor Templates](https://cloud.google.com/security-command-center/docs/manage-model-armor-templates) - Content sanitization and blocking based on configured templates
|
||||
|
||||
## Quick Start
|
||||
### 1. Define Guardrails on your LiteLLM config.yaml
|
||||
|
||||
Define your guardrails under the `guardrails` section
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: openai/gpt-3.5-turbo
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
guardrails:
|
||||
- guardrail_name: model-armor-shield
|
||||
litellm_params:
|
||||
guardrail: model_armor
|
||||
mode: [pre_call, post_call] # Run on both input and output
|
||||
template_id: "your-template-id" # Required: Your Model Armor template ID
|
||||
project_id: "your-project-id" # Your GCP project ID
|
||||
location: "us-central1" # GCP location (default: us-central1)
|
||||
credentials: "path/to/credentials.json" # Path to service account key
|
||||
mask_request_content: true # Enable request content masking
|
||||
mask_response_content: true # Enable response content masking
|
||||
fail_on_error: true # Fail request if Model Armor errors (default: true)
|
||||
default_on: true # Run by default for all requests
|
||||
```
|
||||
|
||||
#### Supported values for `mode`
|
||||
|
||||
- `pre_call` Run **before** LLM call, on **input**
|
||||
- `post_call` Run **after** LLM call, on **input & output**
|
||||
|
||||
### 2. Start LiteLLM Gateway
|
||||
|
||||
|
||||
```shell
|
||||
litellm --config config.yaml --detailed_debug
|
||||
```
|
||||
|
||||
### 3. Test request
|
||||
|
||||
**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
|
||||
|
||||
```shell
|
||||
curl -i http://localhost:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer sk-npnwjPQciVRok5yNZgKmFQ" \
|
||||
-d '{
|
||||
"model": "gpt-3.5-turbo",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hi, my email is test@example.com"}
|
||||
],
|
||||
"guardrails": ["model-armor-shield"]
|
||||
}'
|
||||
```
|
||||
|
||||
## Supported Params
|
||||
|
||||
### Common Params
|
||||
|
||||
- `api_key` - str - Google Cloud service account credentials (optional if using ADC)
|
||||
- `api_base` - str - Custom Model Armor API endpoint (optional)
|
||||
- `default_on` - bool - Whether to run the guardrail by default. Default is `false`.
|
||||
- `mode` - Union[str, list[str]] - Mode to run the guardrail. Either `pre_call` or `post_call`. Default is `pre_call`.
|
||||
|
||||
### Model Armor Specific
|
||||
|
||||
- `template_id` - str - The ID of your Model Armor template (required)
|
||||
- `project_id` - str - Google Cloud project ID (defaults to credentials project)
|
||||
- `location` - str - Google Cloud location/region. Default is `us-central1`
|
||||
- `credentials` - Union[str, dict] - Path to service account JSON file or credentials dictionary
|
||||
- `api_endpoint` - str - Custom API endpoint for Model Armor (optional)
|
||||
- `fail_on_error` - bool - Whether to fail requests if Model Armor encounters errors. Default is `true`
|
||||
- `mask_request_content` - bool - Enable masking of sensitive content in requests. Default is `false`
|
||||
- `mask_response_content` - bool - Enable masking of sensitive content in responses. Default is `false`
|
||||
|
||||
|
||||
## Further Reading
|
||||
|
||||
- [Control Guardrails per API Key](./quick_start#-control-guardrails-per-api-key)
|
||||
|
|
@ -47,6 +47,7 @@
|
|||
"node": ">=16.14"
|
||||
},
|
||||
"overrides": {
|
||||
"webpack-dev-server": ">=5.2.1"
|
||||
"webpack-dev-server": ">=5.2.1",
|
||||
"form-data": ">=4.0.4"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ const sidebars = {
|
|||
"proxy/guardrails/lasso_security",
|
||||
"proxy/guardrails/guardrails_ai",
|
||||
"proxy/guardrails/lakera_ai",
|
||||
"proxy/guardrails/model_armor",
|
||||
"proxy/guardrails/openai_moderation",
|
||||
"proxy/guardrails/pangea",
|
||||
"proxy/guardrails/pii_masking_v2",
|
||||
|
|
@ -411,6 +412,7 @@ const sidebars = {
|
|||
"providers/huggingface_rerank",
|
||||
]
|
||||
},
|
||||
"providers/hyperbolic",
|
||||
"providers/databricks",
|
||||
"providers/deepgram",
|
||||
"providers/watsonx",
|
||||
|
|
@ -448,6 +450,8 @@ const sidebars = {
|
|||
"providers/replicate",
|
||||
"providers/togetherai",
|
||||
"providers/v0",
|
||||
"providers/morph",
|
||||
"providers/lambda_ai",
|
||||
"providers/novita",
|
||||
"providers/voyage",
|
||||
"providers/jina_ai",
|
||||
|
|
@ -624,6 +628,7 @@ const sidebars = {
|
|||
"projects/llm_cord",
|
||||
"projects/pgai",
|
||||
"projects/GPTLocalhost",
|
||||
"projects/HolmesGPT"
|
||||
],
|
||||
},
|
||||
"extras/code_quality",
|
||||
|
|
|
|||
|
|
@ -83,15 +83,25 @@ class CheckBatchCost:
|
|||
)
|
||||
continue
|
||||
|
||||
response = await self.llm_router.aretrieve_batch(
|
||||
model=model_id,
|
||||
batch_id=batch_id,
|
||||
litellm_metadata={
|
||||
"user_api_key_user_id": job.created_by or "default-user-id",
|
||||
"batch_ignore_default_logging": True,
|
||||
},
|
||||
verbose_proxy_logger.info(
|
||||
f"Querying model ID: {model_id} for cost and usage of batch ID: {batch_id}"
|
||||
)
|
||||
|
||||
try:
|
||||
response = await self.llm_router.aretrieve_batch(
|
||||
model=model_id,
|
||||
batch_id=batch_id,
|
||||
litellm_metadata={
|
||||
"user_api_key_user_id": job.created_by or "default-user-id",
|
||||
"batch_ignore_default_logging": True,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.info(
|
||||
f"Skipping job {unified_object_id} because of error querying model ID: {model_id} for cost and usage of batch ID: {batch_id}: {e}"
|
||||
)
|
||||
continue
|
||||
|
||||
## RETRIEVE THE BATCH JOB OUTPUT FILE
|
||||
managed_files_obj = cast(
|
||||
Optional[_PROXY_LiteLLMManagedFiles],
|
||||
|
|
@ -102,6 +112,9 @@ class CheckBatchCost:
|
|||
and response.output_file_id is not None
|
||||
and managed_files_obj is not None
|
||||
):
|
||||
verbose_proxy_logger.info(
|
||||
f"Batch ID: {batch_id} is complete, tracking cost and usage"
|
||||
)
|
||||
# track cost
|
||||
model_file_id_mapping = {
|
||||
response.output_file_id: {model_id: response.output_file_id}
|
||||
|
|
|
|||
|
|
@ -144,22 +144,22 @@ prometheus_initialize_budget_metrics: Optional[bool] = False
|
|||
require_auth_for_metrics_endpoint: Optional[bool] = False
|
||||
argilla_batch_size: Optional[int] = None
|
||||
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
|
||||
gcs_pub_sub_use_v1: Optional[bool] = (
|
||||
False # if you want to use v1 gcs pubsub logged payload
|
||||
)
|
||||
generic_api_use_v1: Optional[bool] = (
|
||||
False # if you want to use v1 generic api logged payload
|
||||
)
|
||||
gcs_pub_sub_use_v1: Optional[
|
||||
bool
|
||||
] = False # if you want to use v1 gcs pubsub logged payload
|
||||
generic_api_use_v1: Optional[
|
||||
bool
|
||||
] = False # if you want to use v1 generic api logged payload
|
||||
argilla_transformation_object: Optional[Dict[str, Any]] = None
|
||||
_async_input_callback: List[Union[str, Callable, CustomLogger]] = (
|
||||
[]
|
||||
) # internal variable - async custom callbacks are routed here.
|
||||
_async_success_callback: List[Union[str, Callable, CustomLogger]] = (
|
||||
[]
|
||||
) # internal variable - async custom callbacks are routed here.
|
||||
_async_failure_callback: List[Union[str, Callable, CustomLogger]] = (
|
||||
[]
|
||||
) # internal variable - async custom callbacks are routed here.
|
||||
_async_input_callback: List[
|
||||
Union[str, Callable, CustomLogger]
|
||||
] = [] # internal variable - async custom callbacks are routed here.
|
||||
_async_success_callback: List[
|
||||
Union[str, Callable, CustomLogger]
|
||||
] = [] # internal variable - async custom callbacks are routed here.
|
||||
_async_failure_callback: List[
|
||||
Union[str, Callable, CustomLogger]
|
||||
] = [] # internal variable - async custom callbacks are routed here.
|
||||
pre_call_rules: List[Callable] = []
|
||||
post_call_rules: List[Callable] = []
|
||||
turn_off_message_logging: Optional[bool] = False
|
||||
|
|
@ -167,18 +167,18 @@ log_raw_request_response: bool = False
|
|||
redact_messages_in_exceptions: Optional[bool] = False
|
||||
redact_user_api_key_info: Optional[bool] = False
|
||||
filter_invalid_headers: Optional[bool] = False
|
||||
add_user_information_to_llm_headers: Optional[bool] = (
|
||||
None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
|
||||
)
|
||||
add_user_information_to_llm_headers: Optional[
|
||||
bool
|
||||
] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
|
||||
store_audit_logs = False # Enterprise feature, allow users to see audit logs
|
||||
### end of callbacks #############
|
||||
|
||||
email: Optional[str] = (
|
||||
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
|
||||
)
|
||||
token: Optional[str] = (
|
||||
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
|
||||
)
|
||||
email: Optional[
|
||||
str
|
||||
] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
|
||||
token: Optional[
|
||||
str
|
||||
] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
|
||||
telemetry = True
|
||||
max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults
|
||||
drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False))
|
||||
|
|
@ -266,15 +266,11 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
|
|||
enable_caching_on_provider_specific_optional_params: bool = (
|
||||
False # feature-flag for caching on optional params - e.g. 'top_k'
|
||||
)
|
||||
caching: bool = (
|
||||
False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
|
||||
)
|
||||
caching_with_models: bool = (
|
||||
False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
|
||||
)
|
||||
cache: Optional[Cache] = (
|
||||
None # cache object <- use this - https://docs.litellm.ai/docs/caching
|
||||
)
|
||||
caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
|
||||
caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
|
||||
cache: Optional[
|
||||
Cache
|
||||
] = None # cache object <- use this - https://docs.litellm.ai/docs/caching
|
||||
default_in_memory_ttl: Optional[float] = None
|
||||
default_redis_ttl: Optional[float] = None
|
||||
default_redis_batch_cache_expiry: Optional[float] = None
|
||||
|
|
@ -282,9 +278,9 @@ model_alias_map: Dict[str, str] = {}
|
|||
model_group_alias_map: Dict[str, str] = {}
|
||||
model_group_settings: Optional["ModelGroupSettings"] = None
|
||||
max_budget: float = 0.0 # set the max budget across all providers
|
||||
budget_duration: Optional[str] = (
|
||||
None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
|
||||
)
|
||||
budget_duration: Optional[
|
||||
str
|
||||
] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
|
||||
default_soft_budget: float = (
|
||||
DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0
|
||||
)
|
||||
|
|
@ -293,15 +289,11 @@ forward_traceparent_to_llm_provider: bool = False
|
|||
|
||||
_current_cost = 0.0 # private variable, used if max budget is set
|
||||
error_logs: Dict = {}
|
||||
add_function_to_prompt: bool = (
|
||||
False # if function calling not supported by api, append function call details to system prompt
|
||||
)
|
||||
add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt
|
||||
client_session: Optional[httpx.Client] = None
|
||||
aclient_session: Optional[httpx.AsyncClient] = None
|
||||
model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks'
|
||||
model_cost_map_url: str = (
|
||||
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
|
||||
)
|
||||
model_cost_map_url: str = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
|
||||
suppress_debug_info = False
|
||||
dynamodb_table_name: Optional[str] = None
|
||||
s3_callback_params: Optional[Dict] = None
|
||||
|
|
@ -329,9 +321,7 @@ prometheus_metrics_config: Optional[List] = None
|
|||
disable_add_prefix_to_prompt: bool = (
|
||||
False # used by anthropic, to disable adding prefix to prompt
|
||||
)
|
||||
disable_copilot_system_to_assistant: bool = (
|
||||
False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
|
||||
)
|
||||
disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
|
||||
public_model_groups: Optional[List[str]] = None
|
||||
public_model_groups_links: Dict[str, str] = {}
|
||||
#### REQUEST PRIORITIZATION #####
|
||||
|
|
@ -339,17 +329,13 @@ priority_reservation: Optional[Dict[str, float]] = None
|
|||
|
||||
|
||||
######## Networking Settings ########
|
||||
use_aiohttp_transport: bool = (
|
||||
True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead.
|
||||
)
|
||||
use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead.
|
||||
aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings
|
||||
disable_aiohttp_transport: bool = False # Set this to true to use httpx instead
|
||||
disable_aiohttp_trust_env: bool = (
|
||||
False # When False, aiohttp will respect HTTP(S)_PROXY env vars
|
||||
)
|
||||
force_ipv4: bool = (
|
||||
False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
|
||||
)
|
||||
force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
|
||||
module_level_aclient = AsyncHTTPHandler(
|
||||
timeout=request_timeout, client_alias="module level aclient"
|
||||
)
|
||||
|
|
@ -363,13 +349,13 @@ fallbacks: Optional[List] = None
|
|||
context_window_fallbacks: Optional[List] = None
|
||||
content_policy_fallbacks: Optional[List] = None
|
||||
allowed_fails: int = 3
|
||||
num_retries_per_request: Optional[int] = (
|
||||
None # for the request overall (incl. fallbacks + model retries)
|
||||
)
|
||||
num_retries_per_request: Optional[
|
||||
int
|
||||
] = None # for the request overall (incl. fallbacks + model retries)
|
||||
####### SECRET MANAGERS #####################
|
||||
secret_manager_client: Optional[Any] = (
|
||||
None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
|
||||
)
|
||||
secret_manager_client: Optional[
|
||||
Any
|
||||
] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
|
||||
_google_kms_resource_name: Optional[str] = None
|
||||
_key_management_system: Optional[KeyManagementSystem] = None
|
||||
_key_management_settings: KeyManagementSettings = KeyManagementSettings()
|
||||
|
|
@ -505,6 +491,7 @@ moonshot_models: List = []
|
|||
v0_models: List = []
|
||||
morph_models: List = []
|
||||
lambda_ai_models: List = []
|
||||
hyperbolic_models: List = []
|
||||
recraft_models: List = []
|
||||
|
||||
def is_bedrock_pricing_only_model(key: str) -> bool:
|
||||
|
|
@ -690,6 +677,8 @@ def add_known_models():
|
|||
morph_models.append(key)
|
||||
elif value.get("litellm_provider") == "lambda_ai":
|
||||
lambda_ai_models.append(key)
|
||||
elif value.get("litellm_provider") == "hyperbolic":
|
||||
hyperbolic_models.append(key)
|
||||
elif value.get("litellm_provider") == "recraft":
|
||||
recraft_models.append(key)
|
||||
|
||||
|
|
@ -850,6 +839,7 @@ models_by_provider: dict = {
|
|||
"v0": v0_models,
|
||||
"morph": morph_models,
|
||||
"lambda_ai": lambda_ai_models,
|
||||
"hyperbolic": hyperbolic_models,
|
||||
"recraft": recraft_models,
|
||||
}
|
||||
|
||||
|
|
@ -1171,6 +1161,9 @@ from .llms.nebius.chat.transformation import NebiusConfig
|
|||
from .llms.dashscope.chat.transformation import DashScopeChatConfig
|
||||
from .llms.moonshot.chat.transformation import MoonshotChatConfig
|
||||
from .llms.v0.chat.transformation import V0ChatConfig
|
||||
from .llms.morph.chat.transformation import MorphChatConfig
|
||||
from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig
|
||||
from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig
|
||||
from .main import * # type: ignore
|
||||
from .integrations import *
|
||||
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
|
||||
|
|
@ -1229,12 +1222,12 @@ from .types.llms.custom_llm import CustomLLMItem
|
|||
from .types.utils import GenericStreamingChunk
|
||||
|
||||
custom_provider_map: List[CustomLLMItem] = []
|
||||
_custom_providers: List[str] = (
|
||||
[]
|
||||
) # internal helper util, used to track names of custom providers
|
||||
disable_hf_tokenizer_download: Optional[bool] = (
|
||||
None # disable huggingface tokenizer download. Defaults to openai clk100
|
||||
)
|
||||
_custom_providers: List[
|
||||
str
|
||||
] = [] # internal helper util, used to track names of custom providers
|
||||
disable_hf_tokenizer_download: Optional[
|
||||
bool
|
||||
] = None # disable huggingface tokenizer download. Defaults to openai clk100
|
||||
global_disable_no_log_param: bool = False
|
||||
|
||||
### PASSTHROUGH ###
|
||||
|
|
|
|||
|
|
@ -279,6 +279,8 @@ LITELLM_CHAT_PROVIDERS = [
|
|||
"dashscope",
|
||||
"moonshot",
|
||||
"v0",
|
||||
"morph",
|
||||
"lambda_ai",
|
||||
]
|
||||
|
||||
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [
|
||||
|
|
@ -408,6 +410,9 @@ openai_compatible_endpoints: List = [
|
|||
"https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
|
||||
"https://api.moonshot.ai/v1",
|
||||
"https://api.v0.dev/v1",
|
||||
"https://api.morphllm.com/v1",
|
||||
"https://api.lambda.ai/v1",
|
||||
"https://api.hyperbolic.xyz/v1",
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -446,6 +451,9 @@ openai_compatible_providers: List = [
|
|||
"dashscope",
|
||||
"moonshot",
|
||||
"v0",
|
||||
"morph",
|
||||
"lambda_ai",
|
||||
"hyperbolic",
|
||||
]
|
||||
openai_text_completion_compatible_providers: List = (
|
||||
[ # providers that support `/v1/completions`
|
||||
|
|
@ -458,6 +466,9 @@ openai_text_completion_compatible_providers: List = (
|
|||
"nebius",
|
||||
"dashscope",
|
||||
"moonshot",
|
||||
"v0",
|
||||
"lambda_ai",
|
||||
"hyperbolic",
|
||||
]
|
||||
)
|
||||
_openai_like_providers: List = [
|
||||
|
|
@ -808,6 +819,7 @@ DEFAULT_CRON_JOB_LOCK_TTL_SECONDS = int(
|
|||
PROXY_BUDGET_RESCHEDULER_MIN_TIME = int(
|
||||
os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)
|
||||
)
|
||||
PROXY_BATCH_POLLING_INTERVAL = int(os.getenv("PROXY_BATCH_POLLING_INTERVAL", 3600))
|
||||
PROXY_BUDGET_RESCHEDULER_MAX_TIME = int(
|
||||
os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -240,6 +240,12 @@ def get_llm_provider( # noqa: PLR0915
|
|||
elif endpoint == "https://api.v0.dev/v1":
|
||||
custom_llm_provider = "v0"
|
||||
dynamic_api_key = get_secret_str("V0_API_KEY")
|
||||
elif endpoint == "https://api.lambda.ai/v1":
|
||||
custom_llm_provider = "lambda_ai"
|
||||
dynamic_api_key = get_secret_str("LAMBDA_API_KEY")
|
||||
elif endpoint == "https://api.hyperbolic.xyz/v1":
|
||||
custom_llm_provider = "hyperbolic"
|
||||
dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY")
|
||||
|
||||
if api_base is not None and not isinstance(api_base, str):
|
||||
raise Exception(
|
||||
|
|
@ -530,7 +536,7 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
|
|||
# DataRobot is OpenAI compatible.
|
||||
(
|
||||
api_base,
|
||||
dynamic_api_key
|
||||
dynamic_api_key,
|
||||
) = litellm.DataRobotConfig()._get_openai_compatible_provider_info(
|
||||
api_base, api_key
|
||||
)
|
||||
|
|
@ -691,6 +697,27 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
|
|||
) = litellm.V0ChatConfig()._get_openai_compatible_provider_info(
|
||||
api_base, api_key
|
||||
)
|
||||
elif custom_llm_provider == "morph":
|
||||
(
|
||||
api_base,
|
||||
dynamic_api_key,
|
||||
) = litellm.MorphChatConfig()._get_openai_compatible_provider_info(
|
||||
api_base, api_key
|
||||
)
|
||||
elif custom_llm_provider == "lambda_ai":
|
||||
(
|
||||
api_base,
|
||||
dynamic_api_key,
|
||||
) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info(
|
||||
api_base, api_key
|
||||
)
|
||||
elif custom_llm_provider == "hyperbolic":
|
||||
(
|
||||
api_base,
|
||||
dynamic_api_key,
|
||||
) = litellm.HyperbolicChatConfig()._get_openai_compatible_provider_info(
|
||||
api_base, api_key
|
||||
)
|
||||
|
||||
if api_base is not None and not isinstance(api_base, str):
|
||||
raise Exception("api base needs to be a string. api_base={}".format(api_base))
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
|||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
_get_httpx_client,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.types.llms.anthropic import (
|
||||
|
|
@ -433,7 +434,9 @@ class AnthropicChatCompletion(BaseLLM):
|
|||
|
||||
else:
|
||||
if client is None or not isinstance(client, HTTPHandler):
|
||||
client = HTTPHandler(timeout=timeout) # type: ignore
|
||||
client = _get_httpx_client(
|
||||
params={"timeout": timeout}
|
||||
)
|
||||
else:
|
||||
client = client
|
||||
|
||||
|
|
|
|||
|
|
@ -278,6 +278,7 @@ def get_azure_ad_token(
|
|||
3. From username and password
|
||||
4. From OIDC token
|
||||
5. From a service principal with secret workflow
|
||||
6. From DefaultAzureCredential
|
||||
|
||||
Args:
|
||||
litellm_params: Dictionary containing authentication parameters
|
||||
|
|
@ -352,18 +353,27 @@ def get_azure_ad_token(
|
|||
azure_tenant_id=tenant_id,
|
||||
scope=scope,
|
||||
)
|
||||
# Try to get token provider from service principal
|
||||
# Try to get token provider from service principal or DefaultAzureCredential
|
||||
elif (
|
||||
azure_ad_token_provider is None
|
||||
and litellm.enable_azure_ad_token_refresh is True
|
||||
):
|
||||
verbose_logger.debug(
|
||||
"Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth"
|
||||
"Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth"
|
||||
)
|
||||
try:
|
||||
azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope)
|
||||
except ValueError:
|
||||
verbose_logger.debug("Azure AD Token Provider could not be used.")
|
||||
|
||||
#########################################################
|
||||
# If litellm.enable_azure_ad_token_refresh is True and no other token provider is available,
|
||||
# try to get DefaultAzureCredential provider
|
||||
#########################################################
|
||||
if azure_ad_token_provider is None and azure_ad_token is None:
|
||||
azure_ad_token_provider = BaseAzureLLM._try_get_default_azure_credential_provider(
|
||||
scope=scope,
|
||||
)
|
||||
|
||||
# Execute the token provider to get the token if available
|
||||
if azure_ad_token_provider and callable(azure_ad_token_provider):
|
||||
|
|
@ -387,6 +397,38 @@ def get_azure_ad_token(
|
|||
|
||||
|
||||
class BaseAzureLLM(BaseOpenAILLM):
|
||||
@staticmethod
|
||||
def _try_get_default_azure_credential_provider(
|
||||
scope: str,
|
||||
) -> Optional[Callable[[], str]]:
|
||||
"""
|
||||
Try to get DefaultAzureCredential provider
|
||||
|
||||
Args:
|
||||
scope: Azure scope for the token
|
||||
|
||||
Returns:
|
||||
Token provider callable if DefaultAzureCredential is enabled and available, None otherwise
|
||||
"""
|
||||
from litellm.types.secret_managers.get_azure_ad_token_provider import (
|
||||
AzureCredentialType,
|
||||
)
|
||||
|
||||
verbose_logger.debug(
|
||||
"Attempting to use DefaultAzureCredential for Azure Auth"
|
||||
)
|
||||
|
||||
try:
|
||||
azure_ad_token_provider = get_azure_ad_token_provider(
|
||||
azure_scope=scope,
|
||||
azure_credential=AzureCredentialType.DefaultAzureCredential,
|
||||
)
|
||||
verbose_logger.debug("Successfully obtained Azure AD token provider using DefaultAzureCredential")
|
||||
return azure_ad_token_provider
|
||||
except Exception as e:
|
||||
verbose_logger.debug(f"DefaultAzureCredential failed: {str(e)}")
|
||||
return None
|
||||
|
||||
def get_azure_openai_client(
|
||||
self,
|
||||
api_key: Optional[str],
|
||||
|
|
|
|||
|
|
@ -132,6 +132,23 @@ class Authenticator:
|
|||
status_code=401,
|
||||
)
|
||||
|
||||
def get_api_base(self) -> Optional[str]:
|
||||
"""
|
||||
Get the API endpoint from the api-key.json file.
|
||||
|
||||
Returns:
|
||||
Optional[str]: The GitHub Copilot API endpoint, or None if not found.
|
||||
"""
|
||||
try:
|
||||
with open(self.api_key_file, "r") as f:
|
||||
api_key_info = json.load(f)
|
||||
endpoints = api_key_info.get("endpoints", {})
|
||||
api_endpoint = endpoints.get("api")
|
||||
return api_endpoint
|
||||
except (IOError, json.JSONDecodeError, KeyError) as e:
|
||||
verbose_logger.warning(f"Error reading API endpoint from file: {str(e)}")
|
||||
return None
|
||||
|
||||
def _refresh_api_key(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Refresh the API key using the access token.
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from typing import Optional, Tuple
|
||||
from typing import Any, Optional, Tuple, cast
|
||||
|
||||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.llms.openai.openai import OpenAIConfig
|
||||
|
|
@ -25,7 +25,7 @@ class GithubCopilotConfig(OpenAIConfig):
|
|||
api_key: Optional[str],
|
||||
custom_llm_provider: str,
|
||||
) -> Tuple[Optional[str], Optional[str], str]:
|
||||
api_base = self.GITHUB_COPILOT_API_BASE
|
||||
dynamic_api_base = self.authenticator.get_api_base() or self.GITHUB_COPILOT_API_BASE
|
||||
try:
|
||||
dynamic_api_key = self.authenticator.get_api_key()
|
||||
except GetAPIKeyError as e:
|
||||
|
|
@ -34,7 +34,7 @@ class GithubCopilotConfig(OpenAIConfig):
|
|||
llm_provider=custom_llm_provider,
|
||||
message=str(e),
|
||||
)
|
||||
return api_base, dynamic_api_key, custom_llm_provider
|
||||
return dynamic_api_base, dynamic_api_key, custom_llm_provider
|
||||
|
||||
def _transform_messages(
|
||||
self,
|
||||
|
|
@ -46,5 +46,5 @@ class GithubCopilotConfig(OpenAIConfig):
|
|||
if not disable_copilot_system_to_assistant:
|
||||
for message in messages:
|
||||
if "role" in message and message["role"] == "system":
|
||||
message["role"] = "assistant"
|
||||
cast(Any, message)["role"] = "assistant"
|
||||
return messages
|
||||
|
|
|
|||
0
litellm/llms/hyperbolic/__init__.py
Normal file
0
litellm/llms/hyperbolic/__init__.py
Normal file
0
litellm/llms/hyperbolic/chat/__init__.py
Normal file
0
litellm/llms/hyperbolic/chat/__init__.py
Normal file
54
litellm/llms/hyperbolic/chat/transformation.py
Normal file
54
litellm/llms/hyperbolic/chat/transformation.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
"""
|
||||
Translate from OpenAI's `/v1/chat/completions` to Hyperbolic's `/v1/chat/completions`
|
||||
"""
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
from ...openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
|
||||
|
||||
class HyperbolicChatConfig(OpenAILikeChatConfig):
|
||||
"""
|
||||
Hyperbolic is OpenAI-compatible with standard endpoints
|
||||
"""
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> Optional[str]:
|
||||
return "hyperbolic"
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self, api_base: Optional[str], api_key: Optional[str]
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
# Hyperbolic is openai compatible, we just need to set the api_base
|
||||
api_base = (
|
||||
api_base
|
||||
or get_secret_str("HYPERBOLIC_API_BASE")
|
||||
or "https://api.hyperbolic.xyz/v1" # Default Hyperbolic API base URL
|
||||
) # type: ignore
|
||||
dynamic_api_key = api_key or get_secret_str("HYPERBOLIC_API_KEY")
|
||||
return api_base, dynamic_api_key
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
"""
|
||||
Hyperbolic supports standard OpenAI parameters
|
||||
Reference: https://docs.hyperbolic.xyz/docs/rest-api
|
||||
"""
|
||||
return [
|
||||
"messages", # Required
|
||||
"model", # Required
|
||||
"stream", # Optional
|
||||
"temperature", # Optional
|
||||
"top_p", # Optional
|
||||
"max_tokens", # Optional
|
||||
"frequency_penalty", # Optional
|
||||
"presence_penalty", # Optional
|
||||
"stop", # Optional
|
||||
"n", # Optional
|
||||
"tools", # Optional
|
||||
"tool_choice", # Optional
|
||||
"response_format", # Optional
|
||||
"seed", # Optional
|
||||
"user", # Optional
|
||||
]
|
||||
0
litellm/llms/lambda_ai/__init__.py
Normal file
0
litellm/llms/lambda_ai/__init__.py
Normal file
0
litellm/llms/lambda_ai/chat/__init__.py
Normal file
0
litellm/llms/lambda_ai/chat/__init__.py
Normal file
31
litellm/llms/lambda_ai/chat/transformation.py
Normal file
31
litellm/llms/lambda_ai/chat/transformation.py
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
"""
|
||||
Translate from OpenAI's `/v1/chat/completions` to Lambda's `/v1/chat/completions`
|
||||
"""
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
from ...openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
|
||||
|
||||
class LambdaAIChatConfig(OpenAILikeChatConfig):
|
||||
"""
|
||||
Lambda AI is OpenAI-compatible with standard endpoints
|
||||
"""
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> Optional[str]:
|
||||
return "lambda_ai"
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self, api_base: Optional[str], api_key: Optional[str]
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
# Lambda AI is openai compatible, we just need to set the api_base
|
||||
api_base = (
|
||||
api_base
|
||||
or get_secret_str("LAMBDA_API_BASE")
|
||||
or "https://api.lambda.ai/v1" # Default Lambda API base URL
|
||||
) # type: ignore
|
||||
dynamic_api_key = api_key or get_secret_str("LAMBDA_API_KEY")
|
||||
return api_base, dynamic_api_key
|
||||
0
litellm/llms/morph/__init__.py
Normal file
0
litellm/llms/morph/__init__.py
Normal file
0
litellm/llms/morph/chat/__init__.py
Normal file
0
litellm/llms/morph/chat/__init__.py
Normal file
46
litellm/llms/morph/chat/transformation.py
Normal file
46
litellm/llms/morph/chat/transformation.py
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
"""
|
||||
Transform request from OpenAI format to Morph format.
|
||||
|
||||
[TODO] Docs: Morph supports the OpenAI API format.
|
||||
https://docs.morphllm.com/quickstart
|
||||
"""
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
from ...openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
|
||||
|
||||
class MorphChatConfig(OpenAILikeChatConfig):
|
||||
"""
|
||||
Transform request from OpenAI format to Morph format.
|
||||
"""
|
||||
|
||||
@property
|
||||
def custom_llm_provider(self) -> Optional[str]:
|
||||
return "morph"
|
||||
|
||||
def _get_openai_compatible_provider_info(
|
||||
self, api_base: Optional[str], api_key: Optional[str]
|
||||
) -> Tuple[Optional[str], Optional[str]]:
|
||||
api_base = (
|
||||
api_base
|
||||
or get_secret_str("MORPH_API_BASE")
|
||||
or "https://api.morphllm.com/v1" # default api base
|
||||
)
|
||||
dynamic_api_key = api_key or get_secret_str("MORPH_API_KEY")
|
||||
return api_base, dynamic_api_key
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list:
|
||||
return [
|
||||
"messages",
|
||||
"model",
|
||||
"stream",
|
||||
]
|
||||
|
||||
def pre_call(self, messages: list, model: str, api_key: str, api_base: str):
|
||||
"""
|
||||
Hook for any pre-processing before the API call.
|
||||
"""
|
||||
return
|
||||
|
|
@ -35,6 +35,7 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti
|
|||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
_get_httpx_client,
|
||||
get_async_httpx_client,
|
||||
)
|
||||
from litellm.types.llms.anthropic import AnthropicThinkingParam
|
||||
|
|
@ -1881,7 +1882,7 @@ class VertexLLM(VertexBase):
|
|||
if isinstance(timeout, float) or isinstance(timeout, int):
|
||||
timeout = httpx.Timeout(timeout)
|
||||
_params["timeout"] = timeout
|
||||
client = HTTPHandler(**_params) # type: ignore
|
||||
client = _get_httpx_client(params=_params)
|
||||
else:
|
||||
client = client
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
|
|||
"seed", # equivalent to random_seed
|
||||
"stream", # equivalent to stream
|
||||
"tools",
|
||||
"tool_choice", # equivalent to tool_choice + tool_choice_options
|
||||
"tool_choice", # equivalent to tool_choice + tool_choice_option
|
||||
"logprobs",
|
||||
"top_logprobs",
|
||||
"n",
|
||||
|
|
@ -61,7 +61,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
|
|||
|
||||
_tool_choice = non_default_params.pop("tool_choice", None)
|
||||
if self.is_tool_choice_option(_tool_choice):
|
||||
optional_params["tool_choice_options"] = _tool_choice
|
||||
optional_params["tool_choice_option"] = _tool_choice
|
||||
elif _tool_choice is not None:
|
||||
optional_params["tool_choice"] = _tool_choice
|
||||
return super().map_openai_params(
|
||||
|
|
|
|||
|
|
@ -3486,13 +3486,13 @@ async def acompletion_with_retries(*args, **kwargs):
|
|||
retry_strategy = kwargs.pop("retry_strategy", "constant_retry")
|
||||
original_function = kwargs.pop("original_function", completion)
|
||||
if retry_strategy == "exponential_backoff_retry":
|
||||
retryer = tenacity.AsyncRetrying(
|
||||
retryer = tenacity.Retrying(
|
||||
wait=tenacity.wait_exponential(multiplier=1, max=10),
|
||||
stop=tenacity.stop_after_attempt(num_retries),
|
||||
reraise=True,
|
||||
)
|
||||
else:
|
||||
retryer = tenacity.AsyncRetrying(
|
||||
retryer = tenacity.Retrying(
|
||||
stop=tenacity.stop_after_attempt(num_retries), reraise=True
|
||||
)
|
||||
return await retryer(original_function, *args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -5417,12 +5417,12 @@
|
|||
"supports_tool_choice": true,
|
||||
"deprecation_date": "2025-01-06"
|
||||
},
|
||||
"groq/qwen-qwq-32b": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"groq/qwen/qwen3-32b": {
|
||||
"max_tokens": 131000,
|
||||
"max_input_tokens": 131000,
|
||||
"max_output_tokens": 131000,
|
||||
"input_cost_per_token": 2.9e-07,
|
||||
"output_cost_per_token": 3.9e-07,
|
||||
"output_cost_per_token": 5.9e-07,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
|
|
@ -11039,6 +11039,16 @@
|
|||
"mode": "chat",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/qwen/qwen-vl-plus": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 2048,
|
||||
"input_cost_per_token": 0.00021,
|
||||
"output_cost_per_token": 0.00063,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": false
|
||||
},
|
||||
"openrouter/switchpoint/router": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -14603,6 +14613,19 @@
|
|||
"supports_tool_choice": false,
|
||||
"supports_response_schema": true
|
||||
},
|
||||
"fireworks_ai/accounts/fireworks/models/kimi-k2-instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 16384,
|
||||
"input_cost_per_token": 0.6e-06,
|
||||
"output_cost_per_token": 2.5e-06,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct"
|
||||
},
|
||||
"fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": {
|
||||
"max_tokens": 16384,
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -14921,13 +14944,478 @@
|
|||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"voyage/voyage-01": {
|
||||
"max_tokens": 4096,
|
||||
"max_input_tokens": 4096,
|
||||
"lambda_ai/deepseek-llama3.3-70b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"lambda_ai/deepseek-r1-0528": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"lambda_ai/deepseek-r1-671b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 8e-07,
|
||||
"output_cost_per_token": 8e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"lambda_ai/deepseek-v3-0324": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/hermes3-405b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 8e-07,
|
||||
"output_cost_per_token": 8e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/hermes3-70b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/hermes3-8b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2.5e-08,
|
||||
"output_cost_per_token": 4e-08,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/lfm-40b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"output_cost_per_token": 0.0,
|
||||
"litellm_provider": "voyage",
|
||||
"mode": "embedding"
|
||||
"output_cost_per_token": 2e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/lfm-7b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2.5e-08,
|
||||
"output_cost_per_token": 4e-08,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama-4-maverick-17b-128e-instruct-fp8": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 1e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama-4-scout-17b-16e-instruct": {
|
||||
"max_tokens": 16384,
|
||||
"max_input_tokens": 16384,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 1e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.1-405b-instruct-fp8": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 8e-07,
|
||||
"output_cost_per_token": 8e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.1-70b-instruct-fp8": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.1-8b-instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2.5e-08,
|
||||
"output_cost_per_token": 4e-08,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.1-nemotron-70b-instruct-fp8": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.2-11b-vision-instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.5e-08,
|
||||
"output_cost_per_token": 2.5e-08,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.2-3b-instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.5e-08,
|
||||
"output_cost_per_token": 2.5e-08,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.3-70b-instruct-fp8": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/qwen25-coder-32b-instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 1e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/qwen3-32b-fp8": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 1e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"hyperbolic/moonshotai/Kimi-K2-Instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/deepseek-ai/DeepSeek-R1-0528": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"output_cost_per_token": 2.5e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/Qwen/Qwen3-235B-A22B": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/deepseek-ai/DeepSeek-V3-0324": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/Qwen/QwQ-32B": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 2e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/deepseek-ai/DeepSeek-R1": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/deepseek-ai/DeepSeek-V3": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 2e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/meta-llama/Llama-3.3-70B-Instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/meta-llama/Llama-3.2-3B-Instruct": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/Qwen/Qwen2.5-72B-Instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/meta-llama/Meta-Llama-3-70B-Instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/meta-llama/Meta-Llama-3.1-8B-Instruct": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/meta-llama/Meta-Llama-3.1-70B-Instruct": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"voyage/voyage-lite-01": {
|
||||
"max_tokens": 4096,
|
||||
|
|
|
|||
|
|
@ -1,25 +1,11 @@
|
|||
model_list:
|
||||
- model_name: gpt-3.5-turbo-allow
|
||||
- model_name: gpt-4o-mini-batch
|
||||
litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model: azure/gpt-4o-mini
|
||||
api_key: os.environ/AZURE_API_KEY_HIDDEN
|
||||
api_base: os.environ/AZURE_API_BASE_HIDDEN
|
||||
model_info:
|
||||
version: 2
|
||||
- model_name: gpt-3.5-turbo-disallow
|
||||
litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
model_info:
|
||||
version: 2
|
||||
- model_name: zapier-byok-provider/openai/*
|
||||
litellm_params:
|
||||
model: openai/*
|
||||
api_base: http://0.0.0.0:8090
|
||||
- model_name: openai/gpt-4o-mini
|
||||
litellm_params:
|
||||
model: openai/gpt-4o-mini
|
||||
|
||||
litellm_settings:
|
||||
model_group_alias: {"gpt-3.5-turbo-custom": "gpt-3.5-turbo-disallow"}
|
||||
model_group_settings:
|
||||
forward_client_headers_to_llm_api:
|
||||
- "gpt-3.5-turbo-allow"
|
||||
- "zapier-byok-provider/openai/*"
|
||||
general_settings:
|
||||
proxy_batch_polling_interval: 10
|
||||
|
|
@ -132,6 +132,7 @@ from litellm.constants import (
|
|||
DEFAULT_MODEL_CREATED_AT_TIME,
|
||||
LITELLM_PROXY_ADMIN_NAME,
|
||||
PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS,
|
||||
PROXY_BATCH_POLLING_INTERVAL,
|
||||
PROXY_BATCH_WRITE_AT,
|
||||
PROXY_BUDGET_RESCHEDULER_MAX_TIME,
|
||||
PROXY_BUDGET_RESCHEDULER_MIN_TIME,
|
||||
|
|
@ -538,7 +539,7 @@ async def proxy_shutdown_event():
|
|||
|
||||
@asynccontextmanager
|
||||
async def proxy_startup_event(app: FastAPI):
|
||||
global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings, proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time, litellm_proxy_admin_name, db_writer_client, store_model_in_db, premium_user, _license_check
|
||||
global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings, proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time, litellm_proxy_admin_name, db_writer_client, store_model_in_db, premium_user, _license_check, proxy_batch_polling_interval
|
||||
import json
|
||||
|
||||
init_verbose_loggers()
|
||||
|
|
@ -940,6 +941,7 @@ litellm_proxy_admin_name = LITELLM_PROXY_ADMIN_NAME
|
|||
ui_access_mode: Union[Literal["admin", "all"], Dict] = "all"
|
||||
proxy_budget_rescheduler_min_time = PROXY_BUDGET_RESCHEDULER_MIN_TIME
|
||||
proxy_budget_rescheduler_max_time = PROXY_BUDGET_RESCHEDULER_MAX_TIME
|
||||
proxy_batch_polling_interval = PROXY_BATCH_POLLING_INTERVAL
|
||||
proxy_batch_write_at = PROXY_BATCH_WRITE_AT
|
||||
litellm_master_key_hash = None
|
||||
disable_spend_logs = False
|
||||
|
|
@ -1697,7 +1699,7 @@ class ProxyConfig:
|
|||
"""
|
||||
Load config values into proxy global state
|
||||
"""
|
||||
global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, health_check_interval, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, callback_settings
|
||||
global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, health_check_interval, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, callback_settings, proxy_batch_polling_interval
|
||||
|
||||
config: dict = await self.get_config(config_file_path=config_file_path)
|
||||
|
||||
|
|
@ -2040,6 +2042,10 @@ class ProxyConfig:
|
|||
proxy_budget_rescheduler_max_time = general_settings.get(
|
||||
"proxy_budget_rescheduler_max_time", proxy_budget_rescheduler_max_time
|
||||
)
|
||||
## BATCH POLLING INTERVAL ##
|
||||
proxy_batch_polling_interval = general_settings.get(
|
||||
"proxy_batch_polling_interval", proxy_batch_polling_interval
|
||||
)
|
||||
## BATCH WRITER ##
|
||||
proxy_batch_write_at = general_settings.get(
|
||||
"proxy_batch_write_at", proxy_batch_write_at
|
||||
|
|
@ -3564,7 +3570,7 @@ class ProxyStartupEvent:
|
|||
scheduler.add_job(
|
||||
check_batch_cost_job.check_batch_cost,
|
||||
"interval",
|
||||
seconds=3600, # these can run infrequently, as batch jobs take time to complete
|
||||
seconds=proxy_batch_polling_interval, # these can run infrequently, as batch jobs take time to complete
|
||||
)
|
||||
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -480,7 +480,7 @@ async def update_sso_settings(sso_config: SSOConfig):
|
|||
elif field_name == "ui_access_mode" and value is not None:
|
||||
|
||||
config["general_settings"]["ui_access_mode"] = value
|
||||
elif field_name in env_var_mapping and value is not None:
|
||||
elif field_name in env_var_mapping and value is not None and len(value) > 0:
|
||||
env_var_name = env_var_mapping[field_name]
|
||||
# Update in config
|
||||
config["environment_variables"][env_var_name] = value
|
||||
|
|
|
|||
|
|
@ -31,18 +31,6 @@ class ForwardClientSideHeadersByModelGroup(CustomLogger):
|
|||
"model_group_alias": model_group_alias,
|
||||
}
|
||||
|
||||
def filter_headers(self, headers: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
Filter the headers to only include the headers that are forwarded to the LLM API.
|
||||
|
||||
E.g. passing 'connection': 'keep-alive' will cause the request to hang, and not be acknowledged on the other side.
|
||||
"""
|
||||
return {
|
||||
k: v
|
||||
for k, v in headers.items()
|
||||
if k.lower() not in ["connection", "content-length"]
|
||||
}
|
||||
|
||||
async def async_pre_call_deployment_hook(
|
||||
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
|
||||
) -> Optional[dict]:
|
||||
|
|
@ -78,7 +66,7 @@ class ForwardClientSideHeadersByModelGroup(CustomLogger):
|
|||
in litellm.model_group_settings.forward_client_headers_to_llm_api
|
||||
):
|
||||
kwargs.setdefault("headers", {}).update(
|
||||
self.filter_headers(kwargs["secret_fields"]["raw_headers"])
|
||||
kwargs["secret_fields"]["raw_headers"]
|
||||
)
|
||||
|
||||
return kwargs
|
||||
|
|
|
|||
|
|
@ -6,7 +6,10 @@ from litellm.types.secret_managers.get_azure_ad_token_provider import (
|
|||
)
|
||||
|
||||
|
||||
def get_azure_ad_token_provider(azure_scope: Optional[str] = None) -> Callable[[], str]:
|
||||
def get_azure_ad_token_provider(
|
||||
azure_scope: Optional[str] = None,
|
||||
azure_credential: Optional[AzureCredentialType] = None,
|
||||
) -> Callable[[], str]:
|
||||
"""
|
||||
Get Azure AD token provider based on Service Principal with Secret workflow.
|
||||
|
||||
|
|
@ -27,6 +30,7 @@ def get_azure_ad_token_provider(azure_scope: Optional[str] = None) -> Callable[[
|
|||
from azure.identity import (
|
||||
CertificateCredential,
|
||||
ClientSecretCredential,
|
||||
DefaultAzureCredential,
|
||||
ManagedIdentityCredential,
|
||||
get_bearer_token_provider,
|
||||
)
|
||||
|
|
@ -37,14 +41,17 @@ def get_azure_ad_token_provider(azure_scope: Optional[str] = None) -> Callable[[
|
|||
or "https://cognitiveservices.azure.com/.default"
|
||||
)
|
||||
|
||||
cred: str = os.environ.get(
|
||||
"AZURE_CREDENTIAL", AzureCredentialType.ClientSecretCredential
|
||||
cred: str = (
|
||||
azure_credential.value if azure_credential else None
|
||||
or os.environ.get("AZURE_CREDENTIAL", AzureCredentialType.ClientSecretCredential)
|
||||
or AzureCredentialType.ClientSecretCredential
|
||||
)
|
||||
credential: Optional[
|
||||
Union[
|
||||
ClientSecretCredential,
|
||||
ManagedIdentityCredential,
|
||||
CertificateCredential,
|
||||
DefaultAzureCredential,
|
||||
Any,
|
||||
]
|
||||
] = None
|
||||
|
|
@ -62,10 +69,15 @@ def get_azure_ad_token_provider(azure_scope: Optional[str] = None) -> Callable[[
|
|||
tenant_id=os.environ["AZURE_TENANT_ID"],
|
||||
certificate_path=os.environ["AZURE_CERTIFICATE_PATH"],
|
||||
)
|
||||
elif cred == AzureCredentialType.DefaultAzureCredential:
|
||||
# DefaultAzureCredential doesn't require explicit environment variables
|
||||
# It automatically discovers credentials from the environment (managed identity, CLI, etc.)
|
||||
credential = DefaultAzureCredential()
|
||||
else:
|
||||
cred_cls = getattr(identity, cred)
|
||||
credential = cred_cls()
|
||||
|
||||
if credential is None:
|
||||
raise ValueError("No credential provided")
|
||||
|
||||
return get_bearer_token_provider(credential, azure_scope)
|
||||
|
|
|
|||
|
|
@ -32,4 +32,5 @@ class RecraftImageEditRequestParams(TypedDict, total=False):
|
|||
substyle: Optional[str] # Additional style specification
|
||||
response_format: Optional[str] # Format of returned images: url or b64_json
|
||||
negative_prompt: Optional[str] # Description of undesired elements
|
||||
controls: Optional[Dict] # Custom parameters to tweak generation process
|
||||
controls: Optional[Dict] # Custom parameters to tweak generation process
|
||||
controls: Optional[Dict]
|
||||
|
|
|
|||
|
|
@ -5,3 +5,4 @@ class AzureCredentialType(str, Enum):
|
|||
ClientSecretCredential = "ClientSecretCredential"
|
||||
ManagedIdentityCredential = "ManagedIdentityCredential"
|
||||
CertificateCredential = "CertificateCredential"
|
||||
DefaultAzureCredential = "DefaultAzureCredential"
|
||||
|
|
|
|||
|
|
@ -2276,6 +2276,8 @@ class LlmProviders(str, Enum):
|
|||
DASHSCOPE = "dashscope"
|
||||
MOONSHOT = "moonshot"
|
||||
V0 = "v0"
|
||||
MORPH = "morph"
|
||||
LAMBDA_AI = "lambda_ai"
|
||||
DEEPSEEK = "deepseek"
|
||||
SAMBANOVA = "sambanova"
|
||||
MARITALK = "maritalk"
|
||||
|
|
@ -2313,6 +2315,7 @@ class LlmProviders(str, Enum):
|
|||
LLAMA = "meta_llama"
|
||||
NSCALE = "nscale"
|
||||
PG_VECTOR = "pg_vector"
|
||||
HYPERBOLIC = "hyperbolic"
|
||||
RECRAFT = "recraft"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6684,6 +6684,8 @@ class ProviderConfigManager:
|
|||
return litellm.DatabricksConfig()
|
||||
elif litellm.LlmProviders.XAI == provider:
|
||||
return litellm.XAIChatConfig()
|
||||
elif litellm.LlmProviders.LAMBDA_AI == provider:
|
||||
return litellm.LambdaAIChatConfig()
|
||||
elif litellm.LlmProviders.LLAMA == provider:
|
||||
return litellm.LlamaAPIConfig()
|
||||
elif litellm.LlmProviders.TEXT_COMPLETION_OPENAI == provider:
|
||||
|
|
@ -6832,6 +6834,8 @@ class ProviderConfigManager:
|
|||
return litellm.MoonshotChatConfig()
|
||||
elif litellm.LlmProviders.V0 == provider:
|
||||
return litellm.V0ChatConfig()
|
||||
elif litellm.LlmProviders.MORPH == provider:
|
||||
return litellm.MorphChatConfig()
|
||||
elif litellm.LlmProviders.BEDROCK == provider:
|
||||
bedrock_route = BedrockModelInfo.get_bedrock_route(model)
|
||||
bedrock_invoke_provider = litellm.BedrockLLM.get_bedrock_invoke_provider(
|
||||
|
|
@ -6880,6 +6884,8 @@ class ProviderConfigManager:
|
|||
return litellm.OpenAIGPTConfig()
|
||||
elif litellm.LlmProviders.NSCALE == provider:
|
||||
return litellm.NscaleConfig()
|
||||
elif litellm.LlmProviders.HYPERBOLIC == provider:
|
||||
return litellm.HyperbolicChatConfig()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -5417,12 +5417,12 @@
|
|||
"supports_tool_choice": true,
|
||||
"deprecation_date": "2025-01-06"
|
||||
},
|
||||
"groq/qwen-qwq-32b": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"groq/qwen/qwen3-32b": {
|
||||
"max_tokens": 131000,
|
||||
"max_input_tokens": 131000,
|
||||
"max_output_tokens": 131000,
|
||||
"input_cost_per_token": 2.9e-07,
|
||||
"output_cost_per_token": 3.9e-07,
|
||||
"output_cost_per_token": 5.9e-07,
|
||||
"litellm_provider": "groq",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
|
|
@ -11039,6 +11039,16 @@
|
|||
"mode": "chat",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/qwen/qwen-vl-plus": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 2048,
|
||||
"input_cost_per_token": 0.00021,
|
||||
"output_cost_per_token": 0.00063,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": false
|
||||
},
|
||||
"openrouter/switchpoint/router": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -14603,6 +14613,19 @@
|
|||
"supports_tool_choice": false,
|
||||
"supports_response_schema": true
|
||||
},
|
||||
"fireworks_ai/accounts/fireworks/models/kimi-k2-instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 16384,
|
||||
"input_cost_per_token": 0.6e-06,
|
||||
"output_cost_per_token": 2.5e-06,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://fireworks.ai/models/fireworks/kimi-k2-instruct"
|
||||
},
|
||||
"fireworks_ai/accounts/fireworks/models/llama-v3p1-405b-instruct": {
|
||||
"max_tokens": 16384,
|
||||
"max_input_tokens": 128000,
|
||||
|
|
@ -14921,13 +14944,478 @@
|
|||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"voyage/voyage-01": {
|
||||
"max_tokens": 4096,
|
||||
"max_input_tokens": 4096,
|
||||
"lambda_ai/deepseek-llama3.3-70b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"lambda_ai/deepseek-r1-0528": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"lambda_ai/deepseek-r1-671b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 8e-07,
|
||||
"output_cost_per_token": 8e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"lambda_ai/deepseek-v3-0324": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 6e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/hermes3-405b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 8e-07,
|
||||
"output_cost_per_token": 8e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/hermes3-70b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/hermes3-8b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2.5e-08,
|
||||
"output_cost_per_token": 4e-08,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/lfm-40b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"output_cost_per_token": 0.0,
|
||||
"litellm_provider": "voyage",
|
||||
"mode": "embedding"
|
||||
"output_cost_per_token": 2e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/lfm-7b": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2.5e-08,
|
||||
"output_cost_per_token": 4e-08,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama-4-maverick-17b-128e-instruct-fp8": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 1e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama-4-scout-17b-16e-instruct": {
|
||||
"max_tokens": 16384,
|
||||
"max_input_tokens": 16384,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 1e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.1-405b-instruct-fp8": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 8e-07,
|
||||
"output_cost_per_token": 8e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.1-70b-instruct-fp8": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.1-8b-instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2.5e-08,
|
||||
"output_cost_per_token": 4e-08,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.1-nemotron-70b-instruct-fp8": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.2-11b-vision-instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.5e-08,
|
||||
"output_cost_per_token": 2.5e-08,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.2-3b-instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.5e-08,
|
||||
"output_cost_per_token": 2.5e-08,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/llama3.3-70b-instruct-fp8": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/qwen25-coder-32b-instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 1e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"lambda_ai/qwen3-32b-fp8": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 5e-08,
|
||||
"output_cost_per_token": 1e-07,
|
||||
"litellm_provider": "lambda_ai",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"hyperbolic/moonshotai/Kimi-K2-Instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/deepseek-ai/DeepSeek-R1-0528": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"output_cost_per_token": 2.5e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/Qwen/Qwen3-235B-A22B": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-06,
|
||||
"output_cost_per_token": 2e-06,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/deepseek-ai/DeepSeek-V3-0324": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/Qwen/QwQ-32B": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 2e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/deepseek-ai/DeepSeek-R1": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 4e-07,
|
||||
"output_cost_per_token": 4e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/deepseek-ai/DeepSeek-V3": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 2e-07,
|
||||
"output_cost_per_token": 2e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/meta-llama/Llama-3.3-70B-Instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/meta-llama/Llama-3.2-3B-Instruct": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/Qwen/Qwen2.5-72B-Instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/meta-llama/Meta-Llama-3-70B-Instruct": {
|
||||
"max_tokens": 131072,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 131072,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/meta-llama/Meta-Llama-3.1-405B-Instruct": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/meta-llama/Meta-Llama-3.1-8B-Instruct": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"hyperbolic/meta-llama/Meta-Llama-3.1-70B-Instruct": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "hyperbolic",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"voyage/voyage-lite-01": {
|
||||
"max_tokens": 4096,
|
||||
|
|
|
|||
119
tests/llm_translation/test_hyperbolic.py
Normal file
119
tests/llm_translation/test_hyperbolic.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
import litellm
|
||||
from litellm import get_llm_provider
|
||||
|
||||
|
||||
def test_get_llm_provider_hyperbolic():
|
||||
"""Test that hyperbolic/ prefix returns the correct provider"""
|
||||
model, provider, _, _ = get_llm_provider(model="hyperbolic/deepseek-v3")
|
||||
assert provider == "hyperbolic"
|
||||
assert model == "deepseek-v3"
|
||||
|
||||
|
||||
def test_hyperbolic_completion_call():
|
||||
"""Test basic completion call structure for Hyperbolic"""
|
||||
# This is primarily a structure test since we don't have actual API keys
|
||||
try:
|
||||
litellm.set_verbose = True
|
||||
response = litellm.completion(
|
||||
model="hyperbolic/qwen-2.5-72b",
|
||||
messages=[{"role": "user", "content": "Hello!"}],
|
||||
mock_response="Hi there!",
|
||||
)
|
||||
assert response is not None
|
||||
except Exception as e:
|
||||
# Expected to fail without valid API key, but should recognize the provider
|
||||
assert "hyperbolic" in str(e).lower() or "api" in str(e).lower()
|
||||
|
||||
|
||||
def test_hyperbolic_config_initialization():
|
||||
"""Test that HyperbolicChatConfig initializes correctly"""
|
||||
from litellm.llms.hyperbolic.chat.transformation import HyperbolicChatConfig
|
||||
|
||||
config = HyperbolicChatConfig()
|
||||
assert config.custom_llm_provider == "hyperbolic"
|
||||
|
||||
|
||||
def test_hyperbolic_get_openai_compatible_provider_info():
|
||||
"""Test API base and key handling"""
|
||||
from litellm.llms.hyperbolic.chat.transformation import HyperbolicChatConfig
|
||||
|
||||
config = HyperbolicChatConfig()
|
||||
|
||||
# Test default API base
|
||||
api_base, api_key = config._get_openai_compatible_provider_info(None, None)
|
||||
assert api_base == "https://api.hyperbolic.xyz/v1"
|
||||
# api_key may be set from environment, so we don't test for None
|
||||
|
||||
# Test custom API base
|
||||
custom_base = "https://custom.hyperbolic.com/v1"
|
||||
api_base, api_key = config._get_openai_compatible_provider_info(custom_base, "test-key")
|
||||
assert api_base == custom_base
|
||||
assert api_key == "test-key"
|
||||
|
||||
|
||||
def test_hyperbolic_in_provider_lists():
|
||||
"""Test that hyperbolic is in all relevant provider lists"""
|
||||
from litellm.constants import (
|
||||
openai_compatible_endpoints,
|
||||
openai_compatible_providers,
|
||||
openai_text_completion_compatible_providers,
|
||||
)
|
||||
|
||||
assert "hyperbolic" in openai_compatible_providers
|
||||
assert "hyperbolic" in openai_text_completion_compatible_providers
|
||||
assert "https://api.hyperbolic.xyz/v1" in openai_compatible_endpoints
|
||||
|
||||
|
||||
def test_hyperbolic_models_configuration():
|
||||
"""Test that Hyperbolic models are properly configured"""
|
||||
import json
|
||||
import os
|
||||
|
||||
# Load model configuration directly from the JSON file
|
||||
json_path = os.path.join(os.path.dirname(__file__), "../../model_prices_and_context_window.json")
|
||||
with open(json_path, 'r') as f:
|
||||
model_data = json.load(f)
|
||||
|
||||
# Test a few key models
|
||||
test_models = [
|
||||
"hyperbolic/deepseek-ai/DeepSeek-V3",
|
||||
"hyperbolic/Qwen/Qwen2.5-Coder-32B-Instruct",
|
||||
"hyperbolic/deepseek-ai/DeepSeek-R1",
|
||||
]
|
||||
|
||||
for model in test_models:
|
||||
assert model in model_data
|
||||
model_info = model_data[model]
|
||||
assert model_info["litellm_provider"] == "hyperbolic"
|
||||
assert model_info["mode"] == "chat"
|
||||
assert "max_tokens" in model_info
|
||||
assert "input_cost_per_token" in model_info
|
||||
assert "output_cost_per_token" in model_info
|
||||
|
||||
|
||||
def test_hyperbolic_supported_params():
|
||||
"""Test that supported OpenAI parameters are correctly configured"""
|
||||
from litellm.llms.hyperbolic.chat.transformation import HyperbolicChatConfig
|
||||
|
||||
config = HyperbolicChatConfig()
|
||||
supported_params = config.get_supported_openai_params("hyperbolic/deepseek-v3")
|
||||
|
||||
# Check for essential parameters
|
||||
assert "messages" in supported_params
|
||||
assert "model" in supported_params
|
||||
assert "stream" in supported_params
|
||||
assert "temperature" in supported_params
|
||||
assert "max_tokens" in supported_params
|
||||
assert "tools" in supported_params
|
||||
assert "tool_choice" in supported_params
|
||||
153
tests/llm_translation/test_lambda_ai.py
Normal file
153
tests/llm_translation/test_lambda_ai.py
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
"""
|
||||
Tests for Lambda AI provider integration
|
||||
"""
|
||||
import os
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import completion
|
||||
from litellm.llms.lambda_ai.chat.transformation import LambdaAIChatConfig
|
||||
|
||||
|
||||
def test_lambda_ai_config_initialization():
|
||||
"""Test LambdaAIChatConfig initializes correctly"""
|
||||
config = LambdaAIChatConfig()
|
||||
assert config.custom_llm_provider == "lambda_ai"
|
||||
|
||||
|
||||
def test_lambda_ai_get_openai_compatible_provider_info():
|
||||
"""Test Lambda AI provider info retrieval"""
|
||||
config = LambdaAIChatConfig()
|
||||
|
||||
# Test with default values (no env vars set)
|
||||
with mock.patch.dict(os.environ, {}, clear=True):
|
||||
api_base, api_key = config._get_openai_compatible_provider_info(None, None)
|
||||
assert api_base == "https://api.lambda.ai/v1"
|
||||
assert api_key is None
|
||||
|
||||
# Test with environment variables
|
||||
with mock.patch.dict(os.environ, {"LAMBDA_API_KEY": "test-key", "LAMBDA_API_BASE": "https://custom.lambda.ai/v1"}):
|
||||
api_base, api_key = config._get_openai_compatible_provider_info(None, None)
|
||||
assert api_base == "https://custom.lambda.ai/v1"
|
||||
assert api_key == "test-key"
|
||||
|
||||
# Test with explicit parameters (should override env vars)
|
||||
with mock.patch.dict(os.environ, {"LAMBDA_API_KEY": "env-key", "LAMBDA_API_BASE": "https://env.lambda.ai/v1"}):
|
||||
api_base, api_key = config._get_openai_compatible_provider_info(
|
||||
"https://param.lambda.ai/v1", "param-key"
|
||||
)
|
||||
assert api_base == "https://param.lambda.ai/v1"
|
||||
assert api_key == "param-key"
|
||||
|
||||
|
||||
def test_get_llm_provider_lambda_ai():
|
||||
"""Test that get_llm_provider correctly identifies Lambda AI"""
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
||||
# Test with lambda_ai/model-name format
|
||||
model, provider, api_key, api_base = get_llm_provider("lambda_ai/llama3.1-8b-instruct")
|
||||
assert model == "llama3.1-8b-instruct"
|
||||
assert provider == "lambda_ai"
|
||||
|
||||
# Test with api_base containing Lambda AI endpoint
|
||||
model, provider, api_key, api_base = get_llm_provider(
|
||||
"llama3.1-8b-instruct", api_base="https://api.lambda.ai/v1"
|
||||
)
|
||||
assert model == "llama3.1-8b-instruct"
|
||||
assert provider == "lambda_ai"
|
||||
assert api_base == "https://api.lambda.ai/v1"
|
||||
|
||||
|
||||
def test_lambda_ai_in_provider_lists():
|
||||
"""Test that Lambda AI is registered in all necessary provider lists"""
|
||||
assert "lambda_ai" in litellm.openai_compatible_providers
|
||||
assert "lambda_ai" in litellm.provider_list
|
||||
assert "https://api.lambda.ai/v1" in litellm.openai_compatible_endpoints
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lambda_ai_completion_call():
|
||||
"""Test completion call with Lambda AI provider (requires LAMBDA_API_KEY)"""
|
||||
# Skip if no API key is available
|
||||
if not os.getenv("LAMBDA_API_KEY"):
|
||||
pytest.skip("LAMBDA_API_KEY not set")
|
||||
|
||||
try:
|
||||
response = await litellm.acompletion(
|
||||
model="lambda_ai/llama3.1-8b-instruct",
|
||||
messages=[{"role": "user", "content": "Hello, this is a test"}],
|
||||
max_tokens=10,
|
||||
)
|
||||
assert response.choices[0].message.content
|
||||
assert response.model
|
||||
assert response.usage
|
||||
except Exception as e:
|
||||
# If the API key is invalid or there's a network issue, that's okay
|
||||
# The important thing is that the provider was recognized
|
||||
if "lambda_ai" not in str(e) and "provider" not in str(e).lower():
|
||||
# Re-raise if it's not a provider-related error
|
||||
raise
|
||||
|
||||
|
||||
def test_lambda_ai_models_configuration():
|
||||
"""Test that Lambda AI models are configured correctly"""
|
||||
from litellm import get_model_info
|
||||
|
||||
# Reload model cost map to pick up local changes
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
# Clear and repopulate lambda_ai_models list after reloading model_cost
|
||||
litellm.lambda_ai_models = []
|
||||
litellm.add_known_models()
|
||||
|
||||
# Some Lambda AI models to test
|
||||
lambda_ai_models = [
|
||||
"lambda_ai/deepseek-llama3.3-70b",
|
||||
"lambda_ai/hermes3-8b",
|
||||
"lambda_ai/llama3.1-8b-instruct",
|
||||
"lambda_ai/llama3.2-11b-vision-instruct",
|
||||
"lambda_ai/qwen25-coder-32b-instruct",
|
||||
]
|
||||
|
||||
for model in lambda_ai_models:
|
||||
model_info = get_model_info(model)
|
||||
assert model_info is not None, f"Model info not found for {model}"
|
||||
assert model_info.get("litellm_provider") == "lambda_ai", f"{model} should have lambda_ai as provider"
|
||||
assert model_info.get("mode") == "chat", f"{model} should be in chat mode"
|
||||
assert model_info.get("supports_function_calling") is True, f"{model} should support function calling"
|
||||
assert model_info.get("supports_system_messages") is True, f"{model} should support system messages"
|
||||
|
||||
# Check vision support for vision models
|
||||
if "vision" in model:
|
||||
assert model_info.get("supports_vision") is True, f"{model} should support vision"
|
||||
|
||||
|
||||
def test_lambda_ai_model_list_populated():
|
||||
"""Test that lambda_ai_models list is populated correctly"""
|
||||
# Ensure we're using local model cost map and repopulate models
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
# Clear and repopulate all model lists after reloading model_cost
|
||||
litellm.lambda_ai_models = []
|
||||
litellm.add_known_models()
|
||||
|
||||
# This should be populated by the add_known_models function
|
||||
assert len(litellm.lambda_ai_models) > 0, "lambda_ai_models list should not be empty"
|
||||
|
||||
# Check that all models in the list are Lambda AI models
|
||||
for model in litellm.lambda_ai_models:
|
||||
assert model.startswith("lambda_ai/"), f"Model {model} should start with 'lambda_ai/'"
|
||||
|
||||
# Check some expected models are in the list
|
||||
expected_models = [
|
||||
"lambda_ai/llama3.1-8b-instruct",
|
||||
"lambda_ai/hermes3-405b",
|
||||
"lambda_ai/deepseek-v3-0324",
|
||||
]
|
||||
|
||||
for model in expected_models:
|
||||
assert model in litellm.lambda_ai_models, f"{model} should be in lambda_ai_models list"
|
||||
108
tests/llm_translation/test_morph.py
Normal file
108
tests/llm_translation/test_morph.py
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
"""Unit tests for Morph provider integration."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.abspath("../..")
|
||||
) # Adds the parent directory to the system path
|
||||
|
||||
import litellm
|
||||
from litellm import MorphChatConfig, get_llm_provider
|
||||
|
||||
# Force model loading
|
||||
litellm.add_known_models()
|
||||
|
||||
|
||||
def test_morph_config_get_provider_info():
|
||||
"""Test that MorphChatConfig returns correct provider info."""
|
||||
config = MorphChatConfig()
|
||||
|
||||
# Test with environment variable
|
||||
with patch.dict(os.environ, {"MORPH_API_KEY": "test-key-from-env"}):
|
||||
api_base, api_key = config._get_openai_compatible_provider_info(None, None)
|
||||
assert api_base == "https://api.morphllm.com/v1"
|
||||
assert api_key == "test-key-from-env"
|
||||
|
||||
# Test with passed api_key
|
||||
api_base, api_key = config._get_openai_compatible_provider_info(None, "direct-key")
|
||||
assert api_base == "https://api.morphllm.com/v1"
|
||||
assert api_key == "direct-key"
|
||||
|
||||
# Test with custom api_base
|
||||
api_base, api_key = config._get_openai_compatible_provider_info("https://custom.morph.com", "key")
|
||||
assert api_base == "https://custom.morph.com"
|
||||
assert api_key == "key"
|
||||
|
||||
|
||||
def test_morph_get_llm_provider():
|
||||
"""Test that get_llm_provider correctly identifies morph models."""
|
||||
# Test with morph/model format
|
||||
_, custom_llm_provider, _, _ = get_llm_provider("morph/morph-v3-large")
|
||||
assert custom_llm_provider == "morph"
|
||||
|
||||
_, custom_llm_provider, _, _ = get_llm_provider("morph/morph-v3-fast")
|
||||
assert custom_llm_provider == "morph"
|
||||
|
||||
|
||||
def test_morph_in_provider_lists():
|
||||
"""Test that morph is included in all necessary provider lists."""
|
||||
import litellm
|
||||
from litellm.constants import openai_compatible_providers, openai_compatible_endpoints
|
||||
|
||||
# Check morph is in openai_compatible_providers
|
||||
assert "morph" in openai_compatible_providers
|
||||
|
||||
# Check morph endpoint is in openai_compatible_endpoints
|
||||
assert "https://api.morphllm.com/v1" in openai_compatible_endpoints
|
||||
|
||||
# Check morph is in provider_list
|
||||
assert "morph" in litellm.provider_list
|
||||
|
||||
# Check models are in model_list after initialization
|
||||
assert all(model in litellm.model_list for model in ["morph/morph-v3-large", "morph/morph-v3-fast"])
|
||||
|
||||
|
||||
def test_morph_model_info():
|
||||
"""Test that morph models have correct configuration."""
|
||||
import litellm
|
||||
model_info = litellm.get_model_info("morph/morph-v3-large")
|
||||
|
||||
assert model_info["litellm_provider"] == "morph"
|
||||
assert model_info["mode"] == "chat"
|
||||
assert model_info["max_tokens"] == 16000
|
||||
assert model_info["max_input_tokens"] == 16000
|
||||
assert model_info["max_output_tokens"] == 16000
|
||||
assert model_info["input_cost_per_token"] == 9e-07 # $0.9/1M tokens
|
||||
assert model_info["output_cost_per_token"] == 1.9e-06 # $1.9/1M tokens
|
||||
assert model_info["supports_function_calling"] is False
|
||||
assert model_info["supports_vision"] is False
|
||||
assert model_info["supports_system_messages"] is True
|
||||
|
||||
|
||||
def test_morph_supported_params():
|
||||
"""Test that MorphChatConfig returns correct supported parameters."""
|
||||
config = MorphChatConfig()
|
||||
supported_params = config.get_supported_openai_params("morph/morph-v3-large")
|
||||
|
||||
expected_params = [
|
||||
"messages",
|
||||
"model",
|
||||
"stream",
|
||||
"temperature",
|
||||
"max_tokens",
|
||||
"tools",
|
||||
"tool_choice",
|
||||
"response_format",
|
||||
]
|
||||
|
||||
assert all(param in supported_params for param in expected_params)
|
||||
|
||||
|
||||
def test_morph_custom_llm_provider():
|
||||
"""Test that morph models are correctly identified."""
|
||||
config = MorphChatConfig()
|
||||
assert config.custom_llm_provider == "morph"
|
||||
|
||||
|
||||
|
|
@ -985,7 +985,7 @@ def test_watsonx_tool_choice():
|
|||
model="gemini-1.5-pro", custom_llm_provider="watsonx", tool_choice="auto"
|
||||
)
|
||||
print(optional_params)
|
||||
assert optional_params["tool_choice_options"] == "auto"
|
||||
assert optional_params["tool_choice_option"] == "auto"
|
||||
|
||||
|
||||
def test_watsonx_text_top_k():
|
||||
|
|
|
|||
|
|
@ -760,7 +760,7 @@ def test_completion_base64(model):
|
|||
pytest.fail(f"An exception occurred - {str(e)}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["claude-3-5-sonnet-latest"])
|
||||
@pytest.mark.parametrize("model", ["claude-3-sonnet-20240229"])
|
||||
def test_completion_function_plus_image(model):
|
||||
litellm.set_verbose = True
|
||||
|
||||
|
|
|
|||
|
|
@ -422,7 +422,7 @@ def test_anthropic_openai_exception():
|
|||
old_azure_key = os.environ["ANTHROPIC_API_KEY"]
|
||||
os.environ.pop("ANTHROPIC_API_KEY")
|
||||
response = completion(
|
||||
model="anthropic/claude-3-5-sonnet-latest",
|
||||
model="anthropic/claude-3-sonnet-20240229",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
)
|
||||
print(f"response: {response}")
|
||||
|
|
@ -495,7 +495,6 @@ def test_completion_bedrock_invalid_role_exception():
|
|||
== "litellm.BadRequestError: Invalid Message passed in {'role': 'very-bad-role', 'content': 'hello'}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="OpenAI exception changed to a generic error")
|
||||
def test_content_policy_exceptionimage_generation_openai():
|
||||
try:
|
||||
|
|
@ -774,15 +773,7 @@ def test_litellm_predibase_exception():
|
|||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider",
|
||||
[
|
||||
"predibase",
|
||||
"vertex_ai_beta",
|
||||
"anthropic",
|
||||
"databricks",
|
||||
"watsonx",
|
||||
"fireworks_ai",
|
||||
],
|
||||
"provider", ["predibase", "vertex_ai_beta", "anthropic", "databricks", "watsonx", "fireworks_ai"]
|
||||
)
|
||||
def test_exception_mapping(provider):
|
||||
"""
|
||||
|
|
@ -835,14 +826,14 @@ def test_fireworks_ai_exception_mapping():
|
|||
2. Text-based rate limit detection (the main issue fixed)
|
||||
3. Generic 400 errors that should NOT be rate limits
|
||||
4. ExceptionCheckers utility function
|
||||
|
||||
|
||||
Related to: https://github.com/BerriAI/litellm/pull/11455
|
||||
Based on Fireworks AI documentation: https://docs.fireworks.ai/tools-sdks/python-client/api-reference
|
||||
"""
|
||||
import litellm
|
||||
from litellm.llms.fireworks_ai.common_utils import FireworksAIException
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers
|
||||
|
||||
|
||||
# Test scenarios covering all important cases
|
||||
test_scenarios = [
|
||||
{
|
||||
|
|
@ -864,63 +855,57 @@ def test_fireworks_ai_exception_mapping():
|
|||
"expected_exception": litellm.BadRequestError,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
# Test each scenario
|
||||
for scenario in test_scenarios:
|
||||
mock_exception = FireworksAIException(
|
||||
status_code=scenario["status_code"], message=scenario["message"], headers={}
|
||||
status_code=scenario["status_code"],
|
||||
message=scenario["message"],
|
||||
headers={}
|
||||
)
|
||||
|
||||
|
||||
try:
|
||||
response = litellm.completion(
|
||||
model="fireworks_ai/llama-v3p1-70b-instruct",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
mock_response=mock_exception,
|
||||
)
|
||||
pytest.fail(
|
||||
f"Expected {scenario['expected_exception'].__name__} to be raised"
|
||||
)
|
||||
pytest.fail(f"Expected {scenario['expected_exception'].__name__} to be raised")
|
||||
except scenario["expected_exception"] as e:
|
||||
if scenario["expected_exception"] == litellm.RateLimitError:
|
||||
assert "rate limit" in str(e).lower() or "429" in str(e)
|
||||
except Exception as e:
|
||||
pytest.fail(
|
||||
f"Expected {scenario['expected_exception'].__name__} but got {type(e).__name__}: {e}"
|
||||
)
|
||||
|
||||
pytest.fail(f"Expected {scenario['expected_exception'].__name__} but got {type(e).__name__}: {e}")
|
||||
|
||||
# Test ExceptionCheckers.is_error_str_rate_limit() method directly
|
||||
|
||||
|
||||
# Test cases that should return True (rate limit detected)
|
||||
rate_limit_strings = [
|
||||
"429 rate limit exceeded",
|
||||
"Rate limit exceeded, please try again later",
|
||||
"Rate limit exceeded, please try again later",
|
||||
"RATE LIMIT ERROR",
|
||||
"Error 429: rate limit",
|
||||
'{"error":{"type":"invalid_request_error","message":"rate limit exceeded, please try again later"}}',
|
||||
"HTTP 429 Too Many Requests",
|
||||
]
|
||||
|
||||
|
||||
for error_str in rate_limit_strings:
|
||||
assert ExceptionCheckers.is_error_str_rate_limit(
|
||||
error_str
|
||||
), f"Should detect rate limit in: {error_str}"
|
||||
|
||||
assert ExceptionCheckers.is_error_str_rate_limit(error_str), f"Should detect rate limit in: {error_str}"
|
||||
|
||||
# Test cases that should return False (not rate limit)
|
||||
non_rate_limit_strings = [
|
||||
"400 Bad Request",
|
||||
"Authentication failed",
|
||||
"Authentication failed",
|
||||
"Invalid model specified",
|
||||
"Context window exceeded",
|
||||
"Internal server error",
|
||||
"",
|
||||
"Some other error message",
|
||||
]
|
||||
|
||||
|
||||
for error_str in non_rate_limit_strings:
|
||||
assert not ExceptionCheckers.is_error_str_rate_limit(
|
||||
error_str
|
||||
), f"Should NOT detect rate limit in: {error_str}"
|
||||
|
||||
assert not ExceptionCheckers.is_error_str_rate_limit(error_str), f"Should NOT detect rate limit in: {error_str}"
|
||||
|
||||
# Test edge cases
|
||||
assert not ExceptionCheckers.is_error_str_rate_limit(None) # type: ignore
|
||||
assert not ExceptionCheckers.is_error_str_rate_limit(42) # type: ignore
|
||||
|
|
@ -1157,7 +1142,6 @@ def test_openai_gateway_timeout_error():
|
|||
"""
|
||||
openai_client = OpenAI()
|
||||
mapped_target = openai_client.chat.completions.with_raw_response # type: ignore
|
||||
|
||||
def _return_exception(*args, **kwargs):
|
||||
import datetime
|
||||
|
||||
|
|
@ -1191,17 +1175,13 @@ def test_openai_gateway_timeout_error():
|
|||
setattr(exception, k, v)
|
||||
raise exception
|
||||
|
||||
try:
|
||||
try:
|
||||
with patch.object(
|
||||
mapped_target,
|
||||
"create",
|
||||
side_effect=_return_exception,
|
||||
):
|
||||
litellm.completion(
|
||||
model="openai/gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "Hello world"}],
|
||||
client=openai_client,
|
||||
)
|
||||
litellm.completion(model="openai/gpt-3.5-turbo", messages=[{"role": "user", "content": "Hello world"}], client=openai_client)
|
||||
pytest.fail("Expected to raise Timeout")
|
||||
except litellm.Timeout as e:
|
||||
assert e.status_code == 504
|
||||
|
|
@ -1397,3 +1377,6 @@ async def test_exception_bubbling_up(sync_mode, stream_mode, model):
|
|||
assert exc_info.value.code == "invalid_value"
|
||||
assert exc_info.value.param is not None
|
||||
assert exc_info.value.type == "invalid_request_error"
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -791,7 +791,7 @@ async def test_watsonx_tool_choice(sync_mode):
|
|||
mock_completion.assert_called_once()
|
||||
print(mock_completion.call_args.kwargs)
|
||||
json_data = json.loads(mock_completion.call_args.kwargs["data"])
|
||||
json_data["tool_choice_options"] == "auto"
|
||||
json_data["tool_choice_option"] == "auto"
|
||||
except Exception as e:
|
||||
print(e)
|
||||
if "The read operation timed out" in str(e):
|
||||
|
|
|
|||
|
|
@ -125,6 +125,12 @@ async def test_router_provider_wildcard_routing():
|
|||
|
||||
print("response 3 = ", response3)
|
||||
|
||||
response4 = await router.acompletion(
|
||||
model="claude-3-5-sonnet-latest",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio()
|
||||
async def test_router_provider_wildcard_routing_regex():
|
||||
"""
|
||||
|
|
@ -1130,7 +1136,7 @@ async def test_aimg_gen_on_router():
|
|||
"api_base": os.getenv("AZURE_SWEDEN_API_BASE"),
|
||||
"api_key": os.getenv("AZURE_SWEDEN_API_KEY"),
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
router = Router(model_list=model_list, num_retries=3)
|
||||
response = await router.aimage_generation(
|
||||
|
|
@ -2781,4 +2787,4 @@ def test_router_get_model_group_info():
|
|||
assert model_group_info is not None
|
||||
assert model_group_info.model_group == "gpt-4"
|
||||
assert model_group_info.input_cost_per_token > 0
|
||||
assert model_group_info.output_cost_per_token > 0
|
||||
assert model_group_info.output_cost_per_token > 0
|
||||
|
|
@ -642,7 +642,6 @@ def test_completion_ollama_hosted_stream():
|
|||
"model",
|
||||
[
|
||||
# "claude-3-5-haiku-20241022",
|
||||
# "claude-2",
|
||||
# "mistral/mistral-small-latest",
|
||||
"openrouter/openai/gpt-4o-mini",
|
||||
],
|
||||
|
|
@ -673,6 +672,7 @@ def test_completion_model_stream(model):
|
|||
pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sync_mode",
|
||||
[True, False],
|
||||
|
|
@ -889,7 +889,6 @@ async def test_completion_gemini_stream_accumulated_json(sync_mode):
|
|||
# return
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
|
||||
|
||||
@pytest.mark.flaky(retries=3, delay=1)
|
||||
def test_completion_mistral_api_mistral_large_function_call_with_streaming():
|
||||
litellm.set_verbose = True
|
||||
|
|
|
|||
|
|
@ -12,7 +12,13 @@ sys.path.insert(
|
|||
) # Adds the parent directory to the system path
|
||||
import litellm
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM, get_azure_ad_token
|
||||
from litellm.secret_managers.get_azure_ad_token_provider import (
|
||||
get_azure_ad_token_provider,
|
||||
)
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.secret_managers.get_azure_ad_token_provider import (
|
||||
AzureCredentialType,
|
||||
)
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
|
||||
|
|
@ -1298,7 +1304,7 @@ def test_get_azure_ad_token_with_token_refresh(setup_mocks, monkeypatch):
|
|||
|
||||
# Verify the debug message was logged
|
||||
setup_mocks["logger"].debug.assert_any_call(
|
||||
"Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth"
|
||||
"Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth"
|
||||
)
|
||||
|
||||
# Verify get_azure_ad_token_provider was called
|
||||
|
|
@ -1325,7 +1331,7 @@ def test_get_azure_ad_token_with_token_refresh_error(setup_mocks):
|
|||
|
||||
# Verify the debug message was logged
|
||||
setup_mocks["logger"].debug.assert_any_call(
|
||||
"Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth"
|
||||
"Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth"
|
||||
)
|
||||
|
||||
# Verify error was logged
|
||||
|
|
@ -1333,8 +1339,8 @@ def test_get_azure_ad_token_with_token_refresh_error(setup_mocks):
|
|||
"Azure AD Token Provider could not be used."
|
||||
)
|
||||
|
||||
# Verify get_azure_ad_token_provider was called
|
||||
setup_mocks["token_provider"].assert_called_once()
|
||||
# Verify get_azure_ad_token_provider was called twice (once for service principal, once for DefaultAzureCredential)
|
||||
assert setup_mocks["token_provider"].call_count == 2
|
||||
|
||||
# Verify the token is None since the provider raised an error
|
||||
assert token is None
|
||||
|
|
@ -1380,3 +1386,101 @@ def test_token_provider_raises_exception(setup_mocks):
|
|||
|
||||
# Verify the error was logged
|
||||
setup_mocks["logger"].error.assert_called()
|
||||
|
||||
|
||||
def test_get_azure_ad_token_provider_with_default_azure_credential():
|
||||
"""
|
||||
Test that get_azure_ad_token_provider correctly uses DefaultAzureCredential
|
||||
when explicitly specified as the credential type. This verifies that the function
|
||||
can dynamically instantiate DefaultAzureCredential and return a working token provider.
|
||||
"""
|
||||
# Mock Azure identity classes
|
||||
with patch('azure.identity.DefaultAzureCredential') as mock_default_cred, \
|
||||
patch('azure.identity.get_bearer_token_provider') as mock_token_provider:
|
||||
|
||||
# Configure mocks
|
||||
mock_credential_instance = MagicMock()
|
||||
mock_default_cred.return_value = mock_credential_instance
|
||||
mock_token_provider.return_value = lambda: "test-default-azure-token"
|
||||
|
||||
# Test with DefaultAzureCredential specified explicitly
|
||||
token_provider = get_azure_ad_token_provider(
|
||||
azure_scope="https://cognitiveservices.azure.com/.default",
|
||||
azure_credential=AzureCredentialType.DefaultAzureCredential
|
||||
)
|
||||
|
||||
# Verify DefaultAzureCredential was instantiated
|
||||
mock_default_cred.assert_called_once_with()
|
||||
|
||||
# Verify get_bearer_token_provider was called with the right parameters
|
||||
mock_token_provider.assert_called_once_with(
|
||||
mock_credential_instance,
|
||||
"https://cognitiveservices.azure.com/.default"
|
||||
)
|
||||
|
||||
# Verify the returned token provider works
|
||||
token = token_provider()
|
||||
assert token == "test-default-azure-token"
|
||||
|
||||
|
||||
def test_get_azure_ad_token_fallback_to_default_azure_credential(setup_mocks, monkeypatch):
|
||||
"""
|
||||
Test that get_azure_ad_token falls back to DefaultAzureCredential when the
|
||||
service principal method fails but token refresh is enabled. This tests the
|
||||
complete fallback flow from service principal to DefaultAzureCredential.
|
||||
"""
|
||||
# Clear environment variables that might interfere
|
||||
monkeypatch.delenv("AZURE_USERNAME", raising=False)
|
||||
monkeypatch.delenv("AZURE_PASSWORD", raising=False)
|
||||
monkeypatch.delenv("AZURE_CLIENT_SECRET", raising=False)
|
||||
monkeypatch.delenv("AZURE_CLIENT_ID", raising=False)
|
||||
monkeypatch.delenv("AZURE_TENANT_ID", raising=False)
|
||||
|
||||
# Reset mocks to ensure clean state
|
||||
setup_mocks["token_provider"].reset_mock()
|
||||
|
||||
# Enable token refresh
|
||||
setup_mocks["litellm"].enable_azure_ad_token_refresh = True
|
||||
|
||||
# Configure get_azure_ad_token_provider to fail first (service principal)
|
||||
# but succeed on second call (DefaultAzureCredential)
|
||||
def mock_token_provider_side_effect(*args, **kwargs):
|
||||
# If called with azure_credential=DefaultAzureCredential, return a working provider
|
||||
if kwargs.get("azure_credential") == AzureCredentialType.DefaultAzureCredential:
|
||||
return lambda: "mock-default-azure-credential-token"
|
||||
# Otherwise (service principal call), return None to simulate failure
|
||||
return None
|
||||
|
||||
setup_mocks["token_provider"].side_effect = mock_token_provider_side_effect
|
||||
|
||||
# Create test parameters with no other auth methods available
|
||||
litellm_params = GenericLiteLLMParams()
|
||||
|
||||
# Call the function
|
||||
token = get_azure_ad_token(litellm_params)
|
||||
|
||||
# Verify the success debug message was logged
|
||||
setup_mocks["logger"].debug.assert_any_call(
|
||||
"Successfully obtained Azure AD token provider using DefaultAzureCredential"
|
||||
)
|
||||
|
||||
# Verify get_azure_ad_token_provider was called twice:
|
||||
# 1. First with just azure_scope (service principal attempt)
|
||||
# 2. Second with azure_credential=DefaultAzureCredential (fallback)
|
||||
assert setup_mocks["token_provider"].call_count == 2
|
||||
|
||||
# Verify the calls were made with expected parameters
|
||||
calls = setup_mocks["token_provider"].call_args_list
|
||||
|
||||
# First call should be service principal attempt (no azure_credential)
|
||||
first_call_kwargs = calls[0][1]
|
||||
assert "azure_scope" in first_call_kwargs
|
||||
assert first_call_kwargs.get("azure_credential") is None
|
||||
|
||||
# Second call should be DefaultAzureCredential attempt
|
||||
second_call_kwargs = calls[1][1]
|
||||
assert "azure_scope" in second_call_kwargs
|
||||
assert second_call_kwargs.get("azure_credential") == AzureCredentialType.DefaultAzureCredential
|
||||
|
||||
# Verify the token is what we expect from our DefaultAzureCredential mock
|
||||
assert token == "mock-default-azure-credential-token"
|
||||
|
|
|
|||
|
|
@ -178,3 +178,14 @@ class TestGitHubCopilotAuthenticator:
|
|||
authenticator._get_device_code.assert_called_once()
|
||||
authenticator._poll_for_access_token.assert_called_once_with("mock-device-code")
|
||||
mock_print.assert_called_once()
|
||||
|
||||
def test_get_api_base_from_file(self, authenticator):
|
||||
"""Test retrieving the API base endpoint from a file."""
|
||||
mock_api_key_data = json.dumps({
|
||||
"token": "mock-api-key",
|
||||
"expires_at": (datetime.now() + timedelta(hours=1)).timestamp(),
|
||||
"endpoints": {"api": "https://api.enterprise.githubcopilot.com"}
|
||||
})
|
||||
with patch("builtins.open", mock_open(read_data=mock_api_key_data)):
|
||||
api_base = authenticator.get_api_base()
|
||||
assert api_base == "https://api.enterprise.githubcopilot.com"
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ def test_github_copilot_config_get_openai_compatible_provider_info():
|
|||
mock_api_key = "gh.test-key-123456789"
|
||||
config.authenticator = MagicMock()
|
||||
config.authenticator.get_api_key.return_value = mock_api_key
|
||||
# Test with dynamic endpoint
|
||||
config.authenticator.get_api_base.return_value = "https://api.enterprise.githubcopilot.com"
|
||||
|
||||
# Test with default values
|
||||
model = "github_copilot/gpt-4"
|
||||
|
|
@ -54,10 +56,24 @@ def test_github_copilot_config_get_openai_compatible_provider_info():
|
|||
custom_llm_provider="github_copilot",
|
||||
)
|
||||
|
||||
assert api_base == "https://api.githubcopilot.com/"
|
||||
assert api_base == "https://api.enterprise.githubcopilot.com"
|
||||
assert dynamic_api_key == mock_api_key
|
||||
assert custom_llm_provider == "github_copilot"
|
||||
|
||||
# Test fallback to default if no dynamic endpoint
|
||||
config.authenticator.get_api_base.return_value = None
|
||||
(
|
||||
api_base,
|
||||
dynamic_api_key,
|
||||
custom_llm_provider,
|
||||
) = config._get_openai_compatible_provider_info(
|
||||
model=model,
|
||||
api_base=None,
|
||||
api_key=None,
|
||||
custom_llm_provider="github_copilot",
|
||||
)
|
||||
assert api_base == "https://api.githubcopilot.com/"
|
||||
|
||||
# Test with authentication failure
|
||||
config.authenticator.get_api_key.side_effect = GetAPIKeyError(
|
||||
message="Failed to get API key",
|
||||
|
|
@ -128,8 +144,8 @@ def test_completion_github_copilot_mock_response(mock_completion, mock_get_api_k
|
|||
|
||||
def test_transform_messages_disable_copilot_system_to_assistant(monkeypatch):
|
||||
"""Test that system messages are converted to assistant unless disable_copilot_system_to_assistant is True."""
|
||||
from litellm.llms.github_copilot.chat.transformation import GithubCopilotConfig
|
||||
import litellm
|
||||
from litellm.llms.github_copilot.chat.transformation import GithubCopilotConfig
|
||||
|
||||
# Save original value
|
||||
original_flag = litellm.disable_copilot_system_to_assistant
|
||||
|
|
|
|||
|
|
@ -1097,28 +1097,3 @@ def test_stream_chunk_builder_thinking_blocks():
|
|||
assert response is not None
|
||||
assert response.choices[0].message.content is not None
|
||||
assert response.choices[0].message.thinking_blocks is not None
|
||||
|
||||
|
||||
from litellm.llms.openai.openai import OpenAIChatCompletion
|
||||
|
||||
|
||||
def throw_retryable_error(*_, **__):
|
||||
raise RuntimeError("BOOM")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retrying() -> None:
|
||||
litellm.num_retries = 10
|
||||
with (
|
||||
patch.object(
|
||||
OpenAIChatCompletion,
|
||||
"make_openai_chat_completion_request",
|
||||
side_effect=throw_retryable_error,
|
||||
) as mock_request,
|
||||
pytest.raises(litellm.InternalServerError, match="LiteLLM Retried: 10 times"),
|
||||
):
|
||||
await litellm.acompletion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
)
|
||||
assert mock_request.call_count >= 10, "Expected retrying to be used"
|
||||
|
|
|
|||
|
|
@ -570,7 +570,7 @@ async def test_proxy_all_models():
|
|||
await chat_completion(
|
||||
session=session,
|
||||
key=LITELLM_MASTER_KEY,
|
||||
model="anthropic/claude-3-5-sonnet-latest",
|
||||
model="anthropic/claude-3-sonnet-20240229",
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ import {
|
|||
teamMemberUpdateCall,
|
||||
Member,
|
||||
teamUpdateCall,
|
||||
getGuardrailsList,
|
||||
} from "@/components/networking";
|
||||
import { Button, Form, Input, Select, message, Tooltip } from "antd";
|
||||
import { InfoCircleOutlined } from "@ant-design/icons";
|
||||
|
|
@ -147,6 +148,7 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
const [mcpAccessGroups, setMcpAccessGroups] = useState<string[]>([]);
|
||||
const [mcpAccessGroupsLoaded, setMcpAccessGroupsLoaded] = useState(false);
|
||||
const [copiedStates, setCopiedStates] = useState<Record<string, boolean>>({})
|
||||
const [guardrailsList, setGuardrailsList] = useState<string[]>([]);
|
||||
|
||||
console.log("userModels in team info", userModels);
|
||||
|
||||
|
|
@ -182,6 +184,23 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const fetchGuardrails = async () => {
|
||||
try {
|
||||
if (!accessToken) return;
|
||||
const response = await getGuardrailsList(accessToken);
|
||||
const guardrailNames = response.guardrails.map(
|
||||
(g: { guardrail_name: string }) => g.guardrail_name
|
||||
);
|
||||
setGuardrailsList(guardrailNames);
|
||||
} catch (error) {
|
||||
console.error("Failed to fetch guardrails:", error);
|
||||
}
|
||||
};
|
||||
|
||||
fetchGuardrails();
|
||||
}, [accessToken]);
|
||||
|
||||
const handleMemberCreate = async (values: any) => {
|
||||
try {
|
||||
if (accessToken == null) return;
|
||||
|
|
@ -673,13 +692,14 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
|
|||
<Select
|
||||
mode="tags"
|
||||
placeholder="Select or enter guardrails"
|
||||
options={guardrailsList.map(name => ({ value: name, label: name }))}
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item label="Vector Stores" name="vector_stores">
|
||||
<VectorStoreSelector
|
||||
onChange={(values) =>
|
||||
form.setFieldValue("vector_stores", values)
|
||||
form.setFieldValue("vector_stores", values)
|
||||
}
|
||||
value={form.getFieldValue("vector_stores")}
|
||||
accessToken={accessToken || ""}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue