mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
Merge branch 'BerriAI:main' into main
This commit is contained in:
TensorNull
GitHub
commit
159cfe5b4c
250 changed files with 5847 additions and 1918 deletions
|
|
@ -83,8 +83,8 @@ jobs:
|
|||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3==1.34.34"
|
||||
pip install "aioboto3==12.3.0"
|
||||
pip install "boto3==1.36.0"
|
||||
pip install "aioboto3==13.4.0"
|
||||
pip install langchain
|
||||
pip install lunary==0.2.5
|
||||
pip install "azure-identity==1.16.1"
|
||||
|
|
@ -208,8 +208,8 @@ jobs:
|
|||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3==1.34.34"
|
||||
pip install "aioboto3==12.3.0"
|
||||
pip install "boto3==1.36.0"
|
||||
pip install "aioboto3==13.4.0"
|
||||
pip install langchain
|
||||
pip install lunary==0.2.5
|
||||
pip install "azure-identity==1.16.1"
|
||||
|
|
@ -315,8 +315,8 @@ jobs:
|
|||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3==1.34.34"
|
||||
pip install "aioboto3==12.3.0"
|
||||
pip install "boto3==1.36.0"
|
||||
pip install "aioboto3==13.4.0"
|
||||
pip install langchain
|
||||
pip install lunary==0.2.5
|
||||
pip install "azure-identity==1.16.1"
|
||||
|
|
@ -590,8 +590,8 @@ jobs:
|
|||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3==1.34.34"
|
||||
pip install "aioboto3==12.3.0"
|
||||
pip install "boto3==1.36.0"
|
||||
pip install "aioboto3==13.4.0"
|
||||
pip install langchain
|
||||
pip install lunary==0.2.5
|
||||
pip install "azure-identity==1.16.1"
|
||||
|
|
@ -816,7 +816,7 @@ jobs:
|
|||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "respx==0.22.0"
|
||||
pip install "pydantic==2.10.2"
|
||||
pip install "boto3==1.34.34"
|
||||
pip install "boto3==1.36.0"
|
||||
# Run pytest and generate JUnit XML report
|
||||
- run:
|
||||
name: Run tests
|
||||
|
|
@ -1238,7 +1238,7 @@ jobs:
|
|||
pip install aiohttp
|
||||
pip install openai
|
||||
pip install click
|
||||
pip install "boto3==1.34.34"
|
||||
pip install "boto3==1.36.0"
|
||||
pip install jinja2
|
||||
pip install "tokenizers==0.20.0"
|
||||
pip install "uvloop==0.21.0"
|
||||
|
|
@ -1507,8 +1507,8 @@ jobs:
|
|||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3==1.34.34"
|
||||
pip install "aioboto3==12.3.0"
|
||||
pip install "boto3==1.36.0"
|
||||
pip install "aioboto3==13.4.0"
|
||||
pip install langchain
|
||||
pip install "langfuse>=2.0.0"
|
||||
pip install "logfire==0.29.0"
|
||||
|
|
@ -1663,8 +1663,8 @@ jobs:
|
|||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3==1.34.34"
|
||||
pip install "aioboto3==12.3.0"
|
||||
pip install "boto3==1.36.0"
|
||||
pip install "aioboto3==13.4.0"
|
||||
pip install langchain
|
||||
pip install "langchain_mcp_adapters==0.0.5"
|
||||
pip install "langfuse>=2.0.0"
|
||||
|
|
@ -1804,8 +1804,8 @@ jobs:
|
|||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3==1.34.34"
|
||||
pip install "aioboto3==12.3.0"
|
||||
pip install "boto3==1.36.0"
|
||||
pip install "aioboto3==13.4.0"
|
||||
pip install langchain
|
||||
pip install "langfuse>=2.0.0"
|
||||
pip install "logfire==0.29.0"
|
||||
|
|
@ -2406,7 +2406,7 @@ jobs:
|
|||
pip install "pytest==7.3.1"
|
||||
pip install "pytest-mock==3.12.0"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install "boto3==1.34.34"
|
||||
pip install "boto3==1.36.0"
|
||||
pip install mypy
|
||||
pip install pyarrow
|
||||
pip install numpydoc
|
||||
|
|
|
|||
2
.github/workflows/test-litellm.yml
vendored
2
.github/workflows/test-litellm.yml
vendored
|
|
@ -7,7 +7,7 @@ on:
|
|||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
|
|||
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -86,6 +86,7 @@ litellm/proxy/db/migrations/0_init/migration.sql
|
|||
litellm/proxy/db/migrations/*
|
||||
litellm/proxy/migrations/*config.yaml
|
||||
litellm/proxy/migrations/*
|
||||
litellm/proxy/to_delete_loadtest_work/*
|
||||
config.yaml
|
||||
tests/litellm/litellm_core_utils/llm_cost_calc/log.txt
|
||||
tests/test_custom_dir/*
|
||||
|
|
|
|||
10
README.md
10
README.md
|
|
@ -374,6 +374,8 @@ We welcome contributions to LiteLLM! Whether you're fixing bugs, adding features
|
|||
|
||||
## Quick Start for Contributors
|
||||
|
||||
This requires poetry to be installed.
|
||||
|
||||
```bash
|
||||
git clone https://github.com/BerriAI/litellm.git
|
||||
cd litellm
|
||||
|
|
@ -381,6 +383,7 @@ make install-dev # Install development dependencies
|
|||
make format # Format your code
|
||||
make lint # Run all linting checks
|
||||
make test-unit # Run unit tests
|
||||
make format-check # Check formatting only
|
||||
```
|
||||
|
||||
For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
|
||||
|
|
@ -396,11 +399,6 @@ Our automated checks include:
|
|||
- **Circular import detection**
|
||||
- **Import safety checks**
|
||||
|
||||
Run all checks locally:
|
||||
```bash
|
||||
make lint # Run all linting (matches CI)
|
||||
make format-check # Check formatting only
|
||||
```
|
||||
|
||||
All these checks must pass before your PR can be merged.
|
||||
|
||||
|
|
@ -442,7 +440,7 @@ All these checks must pass before your PR can be merged.
|
|||
1. (In root) create virtual environment `python -m venv .venv`
|
||||
2. Activate virtual environment `source .venv/bin/activate`
|
||||
3. Install dependencies `pip install -e ".[all]"`
|
||||
4. Start proxy backend `uvicorn litellm.proxy.proxy_server:app --host localhost --port 4000 --reload`
|
||||
4. Start proxy backend `python3 /path/to/litellm/proxy_cli.py`
|
||||
|
||||
### Frontend
|
||||
1. Navigate to `ui/litellm-dashboard`
|
||||
|
|
|
|||
266
docs/my-website/docs/providers/azure_ai_img.md
Normal file
266
docs/my-website/docs/providers/azure_ai_img.md
Normal file
|
|
@ -0,0 +1,266 @@
|
|||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# Azure AI Image Generation
|
||||
|
||||
Azure AI provides powerful image generation capabilities using FLUX models from Black Forest Labs to create high-quality images from text descriptions.
|
||||
|
||||
## Overview
|
||||
|
||||
| Property | Details |
|
||||
|----------|---------|
|
||||
| Description | Azure AI Image Generation uses FLUX models to generate high-quality images from text descriptions. |
|
||||
| Provider Route on LiteLLM | `azure_ai/` |
|
||||
| Provider Doc | [Azure AI FLUX Models ↗](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659) |
|
||||
| Supported Operations | [`/images/generations`](#image-generation) |
|
||||
|
||||
## Setup
|
||||
|
||||
### API Key & Base URL
|
||||
|
||||
```python showLineNumbers
|
||||
# Set your Azure AI API credentials
|
||||
import os
|
||||
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
|
||||
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://your-endpoint.eastus2.inference.ai.azure.com/
|
||||
```
|
||||
|
||||
Get your API key and endpoint from [Azure AI Studio](https://ai.azure.com/).
|
||||
|
||||
## Supported Models
|
||||
|
||||
| Model Name | Description | Cost per Image |
|
||||
|------------|-------------|----------------|
|
||||
| `azure_ai/FLUX-1.1-pro` | Latest FLUX 1.1 Pro model for high-quality image generation | $0.04 |
|
||||
| `azure_ai/FLUX.1-Kontext-pro` | FLUX 1 Kontext Pro model with enhanced context understanding | $0.04 |
|
||||
|
||||
## Image Generation
|
||||
|
||||
### Usage - LiteLLM Python SDK
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="basic" label="Basic Usage">
|
||||
|
||||
```python showLineNumbers title="Basic Image Generation"
|
||||
import litellm
|
||||
import os
|
||||
|
||||
# Set your API credentials
|
||||
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
|
||||
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
|
||||
|
||||
# Generate a single image
|
||||
response = litellm.image_generation(
|
||||
model="azure_ai/FLUX.1-Kontext-pro",
|
||||
prompt="A cute baby sea otter swimming in crystal clear water",
|
||||
api_base=os.environ["AZURE_AI_API_BASE"],
|
||||
api_key=os.environ["AZURE_AI_API_KEY"]
|
||||
)
|
||||
|
||||
print(response.data[0].url)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="flux11" label="FLUX 1.1 Pro">
|
||||
|
||||
```python showLineNumbers title="FLUX 1.1 Pro Image Generation"
|
||||
import litellm
|
||||
import os
|
||||
|
||||
# Set your API credentials
|
||||
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
|
||||
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
|
||||
|
||||
# Generate image with FLUX 1.1 Pro
|
||||
response = litellm.image_generation(
|
||||
model="azure_ai/FLUX-1.1-pro",
|
||||
prompt="A futuristic cityscape at night with neon lights and flying cars",
|
||||
api_base=os.environ["AZURE_AI_API_BASE"],
|
||||
api_key=os.environ["AZURE_AI_API_KEY"]
|
||||
)
|
||||
|
||||
print(response.data[0].url)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="async" label="Async Usage">
|
||||
|
||||
```python showLineNumbers title="Async Image Generation"
|
||||
import litellm
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
async def generate_image():
|
||||
# Set your API credentials
|
||||
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
|
||||
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
|
||||
|
||||
# Generate image asynchronously
|
||||
response = await litellm.aimage_generation(
|
||||
model="azure_ai/FLUX.1-Kontext-pro",
|
||||
prompt="A beautiful sunset over mountains with vibrant colors",
|
||||
api_base=os.environ["AZURE_AI_API_BASE"],
|
||||
api_key=os.environ["AZURE_AI_API_KEY"],
|
||||
n=1,
|
||||
)
|
||||
|
||||
print(response.data[0].url)
|
||||
return response
|
||||
|
||||
# Run the async function
|
||||
asyncio.run(generate_image())
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="advanced" label="Advanced Parameters">
|
||||
|
||||
```python showLineNumbers title="Advanced Image Generation with Parameters"
|
||||
import litellm
|
||||
import os
|
||||
|
||||
# Set your API credentials
|
||||
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
|
||||
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
|
||||
|
||||
# Generate image with additional parameters
|
||||
response = litellm.image_generation(
|
||||
model="azure_ai/FLUX-1.1-pro",
|
||||
prompt="A majestic dragon soaring over a medieval castle at dawn",
|
||||
api_base=os.environ["AZURE_AI_API_BASE"],
|
||||
api_key=os.environ["AZURE_AI_API_KEY"],
|
||||
n=1,
|
||||
size="1024x1024",
|
||||
quality="standard"
|
||||
)
|
||||
|
||||
for image in response.data:
|
||||
print(f"Generated image URL: {image.url}")
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
### Usage - LiteLLM Proxy Server
|
||||
|
||||
#### 1. Configure your config.yaml
|
||||
|
||||
```yaml showLineNumbers title="Azure AI Image Generation Configuration"
|
||||
model_list:
|
||||
- model_name: azure-flux-kontext
|
||||
litellm_params:
|
||||
model: azure_ai/FLUX.1-Kontext-pro
|
||||
api_key: os.environ/AZURE_AI_API_KEY
|
||||
api_base: os.environ/AZURE_AI_API_BASE
|
||||
model_info:
|
||||
mode: image_generation
|
||||
|
||||
- model_name: azure-flux-11-pro
|
||||
litellm_params:
|
||||
model: azure_ai/FLUX-1.1-pro
|
||||
api_key: os.environ/AZURE_AI_API_KEY
|
||||
api_base: os.environ/AZURE_AI_API_BASE
|
||||
model_info:
|
||||
mode: image_generation
|
||||
|
||||
general_settings:
|
||||
master_key: sk-1234
|
||||
```
|
||||
|
||||
#### 2. Start LiteLLM Proxy Server
|
||||
|
||||
```bash showLineNumbers title="Start LiteLLM Proxy Server"
|
||||
litellm --config /path/to/config.yaml
|
||||
|
||||
# RUNNING on http://0.0.0.0:4000
|
||||
```
|
||||
|
||||
#### 3. Make requests with OpenAI Python SDK
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="openai-sdk" label="OpenAI SDK">
|
||||
|
||||
```python showLineNumbers title="Azure AI Image Generation via Proxy - OpenAI SDK"
|
||||
from openai import OpenAI
|
||||
|
||||
# Initialize client with your proxy URL
|
||||
client = OpenAI(
|
||||
base_url="http://localhost:4000", # Your proxy URL
|
||||
api_key="sk-1234" # Your proxy API key
|
||||
)
|
||||
|
||||
# Generate image with FLUX Kontext Pro
|
||||
response = client.images.generate(
|
||||
model="azure-flux-kontext",
|
||||
prompt="A serene Japanese garden with cherry blossoms and a peaceful pond",
|
||||
n=1,
|
||||
size="1024x1024"
|
||||
)
|
||||
|
||||
print(response.data[0].url)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="litellm-sdk" label="LiteLLM SDK">
|
||||
|
||||
```python showLineNumbers title="Azure AI Image Generation via Proxy - LiteLLM SDK"
|
||||
import litellm
|
||||
|
||||
# Configure LiteLLM to use your proxy
|
||||
response = litellm.image_generation(
|
||||
model="litellm_proxy/azure-flux-11-pro",
|
||||
prompt="A cyberpunk warrior in a neon-lit alleyway",
|
||||
api_base="http://localhost:4000",
|
||||
api_key="sk-1234"
|
||||
)
|
||||
|
||||
print(response.data[0].url)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="curl" label="cURL">
|
||||
|
||||
```bash showLineNumbers title="Azure AI Image Generation via Proxy - cURL"
|
||||
curl --location 'http://localhost:4000/v1/images/generations' \
|
||||
--header 'Content-Type: application/json' \
|
||||
--header 'Authorization: Bearer sk-1234' \
|
||||
--data '{
|
||||
"model": "azure-flux-kontext",
|
||||
"prompt": "A cozy coffee shop interior with warm lighting and rustic wooden furniture",
|
||||
"n": 1,
|
||||
"size": "1024x1024"
|
||||
}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
## Supported Parameters
|
||||
|
||||
Azure AI Image Generation supports the following OpenAI-compatible parameters:
|
||||
|
||||
| Parameter | Type | Description | Default | Example |
|
||||
|-----------|------|-------------|---------|---------|
|
||||
| `prompt` | string | Text description of the image to generate | Required | `"A sunset over the ocean"` |
|
||||
| `model` | string | The FLUX model to use for generation | Required | `"azure_ai/FLUX.1-Kontext-pro"` |
|
||||
| `n` | integer | Number of images to generate (1-4) | `1` | `2` |
|
||||
| `size` | string | Image dimensions | `"1024x1024"` | `"512x512"`, `"1024x1024"` |
|
||||
| `api_base` | string | Your Azure AI endpoint URL | Required | `"https://your-endpoint.eastus2.inference.ai.azure.com/"` |
|
||||
| `api_key` | string | Your Azure AI API key | Required | Environment variable or direct value |
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. Create an account at [Azure AI Studio](https://ai.azure.com/)
|
||||
2. Deploy a FLUX model in your Azure AI Studio workspace
|
||||
3. Get your API key and endpoint from the deployment details
|
||||
4. Set your `AZURE_AI_API_KEY` and `AZURE_AI_API_BASE` environment variables
|
||||
5. Start generating images using LiteLLM
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [Azure AI Studio Documentation](https://docs.microsoft.com/en-us/azure/ai-services/)
|
||||
- [FLUX Models Announcement](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659)
|
||||
|
|
@ -584,6 +584,150 @@ Same as [Anthropic API response](../providers/anthropic#usage---thinking--reason
|
|||
Same as [Anthropic API response](../providers/anthropic#usage---thinking--reasoning_content).
|
||||
|
||||
|
||||
## Usage - Anthropic Beta Features
|
||||
|
||||
LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic-beta` header. This enables access to experimental features like:
|
||||
|
||||
- **1M Context Window** - Up to 1 million tokens of context (Claude Sonnet 4)
|
||||
- **Computer Use Tools** - AI that can interact with computer interfaces
|
||||
- **Token-Efficient Tools** - More efficient tool usage patterns
|
||||
- **Extended Output** - Up to 128K output tokens
|
||||
- **Enhanced Thinking** - Advanced reasoning capabilities
|
||||
|
||||
### Supported Beta Features
|
||||
|
||||
| Beta Feature | Header Value | Compatible Models | Description |
|
||||
|--------------|-------------|------------------|-------------|
|
||||
| 1M Context Window | `context-1m-2025-08-07` | Claude Sonnet 4 | Enable 1 million token context window |
|
||||
| Computer Use (Latest) | `computer-use-2025-01-24` | Claude 3.7 Sonnet | Latest computer use tools |
|
||||
| Computer Use (Legacy) | `computer-use-2024-10-22` | Claude 3.5 Sonnet v2 | Computer use tools for Claude 3.5 |
|
||||
| Token-Efficient Tools | `token-efficient-tools-2025-02-19` | Claude 3.7 Sonnet | More efficient tool usage |
|
||||
| Interleaved Thinking | `interleaved-thinking-2025-05-14` | Claude 4 models | Enhanced thinking capabilities |
|
||||
| Extended Output | `output-128k-2025-02-19` | Claude 3.7 Sonnet | Up to 128K output tokens |
|
||||
| Developer Thinking | `dev-full-thinking-2025-05-14` | Claude 4 models | Raw thinking mode for developers |
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
**Single Beta Feature**
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
import os
|
||||
|
||||
# set env
|
||||
os.environ["AWS_ACCESS_KEY_ID"] = ""
|
||||
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
|
||||
os.environ["AWS_REGION_NAME"] = ""
|
||||
|
||||
# Use 1M context window with Claude Sonnet 4
|
||||
response = completion(
|
||||
model="bedrock/anthropic.claude-sonnet-4-20250115-v1:0",
|
||||
messages=[{"role": "user", "content": "Hello! Testing 1M context window."}],
|
||||
max_tokens=100,
|
||||
extra_headers={
|
||||
"anthropic-beta": "context-1m-2025-08-07" # 👈 Enable 1M context
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
**Multiple Beta Features**
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
# Combine multiple beta features (comma-separated)
|
||||
response = completion(
|
||||
model="bedrock/converse/anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
messages=[{"role": "user", "content": "Testing multiple beta features"}],
|
||||
max_tokens=100,
|
||||
extra_headers={
|
||||
"anthropic-beta": "computer-use-2024-10-22,context-1m-2025-08-07"
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
**Computer Use Tools with Beta Features**
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
# Computer use tools automatically add computer-use-2024-10-22
|
||||
# You can add additional beta features
|
||||
response = completion(
|
||||
model="bedrock/converse/anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
messages=[{"role": "user", "content": "Take a screenshot"}],
|
||||
tools=[{
|
||||
"type": "computer_20241022",
|
||||
"name": "computer",
|
||||
"display_width_px": 1920,
|
||||
"display_height_px": 1080
|
||||
}],
|
||||
extra_headers={
|
||||
"anthropic-beta": "context-1m-2025-08-07" # Additional beta feature
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
**Set on YAML Config**
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: claude-sonnet-4-1m
|
||||
litellm_params:
|
||||
model: bedrock/anthropic.claude-sonnet-4-20250115-v1:0
|
||||
extra_headers:
|
||||
anthropic-beta: "context-1m-2025-08-07" # 👈 Enable 1M context
|
||||
|
||||
- model_name: claude-computer-use
|
||||
litellm_params:
|
||||
model: bedrock/converse/anthropic.claude-3-5-sonnet-20241022-v2:0
|
||||
extra_headers:
|
||||
anthropic-beta: "computer-use-2024-10-22,context-1m-2025-08-07"
|
||||
|
||||
general_settings:
|
||||
forward_client_headers_to_llm_api: true # 👈 Required for client-side header forwarding
|
||||
```
|
||||
|
||||
**Set on Request**
|
||||
|
||||
```python
|
||||
import openai
|
||||
|
||||
client = openai.OpenAI(
|
||||
api_key="anything",
|
||||
base_url="http://0.0.0.0:4000"
|
||||
)
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="claude-sonnet-4-1m",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": "Testing 1M context window"
|
||||
}],
|
||||
extra_headers={
|
||||
"anthropic-beta": "context-1m-2025-08-07"
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
:::info
|
||||
**For client-side header forwarding**: When using the proxy and sending `anthropic-beta` headers from the client (like the OpenAI SDK), you need to enable `forward_client_headers_to_llm_api: true` in your proxy's `general_settings`. This tells the proxy to extract headers from HTTP requests and forward them to the underlying LLM provider.
|
||||
:::
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
:::info
|
||||
|
||||
Beta features may require special access or permissions in your AWS account. Some features are only available in specific AWS regions. Check the [AWS Bedrock documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html) for availability and access requirements.
|
||||
|
||||
:::
|
||||
|
||||
|
||||
## Usage - Structured Output / JSON mode
|
||||
|
||||
<Tabs>
|
||||
|
|
|
|||
|
|
@ -104,6 +104,52 @@ Here's how to call an OpenAI-Compatible Endpoint with the LiteLLM Proxy Server
|
|||
|
||||
</Tabs>
|
||||
|
||||
## Reasoning Effort
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="sdk" label="SDK">
|
||||
|
||||
```python
|
||||
from litellm import completion
|
||||
|
||||
response = completion(
|
||||
model="hosted_vllm/gpt-oss-120b",
|
||||
messages=[{"role": "user", "content": "whats 2 + 2"}],
|
||||
reasoning_effort="high",
|
||||
api_base="https://hosted-vllm-api.co",
|
||||
)
|
||||
print(response)
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="proxy" label="PROXY">
|
||||
|
||||
1. Setup config.yaml
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-oss-120b
|
||||
litellm_params:
|
||||
model: hosted_vllm/gpt-oss-120b
|
||||
api_base: https://hosted-vllm-api.co
|
||||
```
|
||||
|
||||
2. Start the proxy
|
||||
|
||||
```bash
|
||||
litellm --config /path/to/config.yaml
|
||||
```
|
||||
|
||||
3. Test it!
|
||||
|
||||
```bash
|
||||
curl http://0.0.0.0:4000/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model": "gpt-oss-120b", "messages": [{"role": "user", "content": "whats 2 + 2"}], "reasoning_effort": "high"}'
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
||||
## Embeddings
|
||||
|
||||
|
|
|
|||
|
|
@ -127,6 +127,8 @@ CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]
|
|||
|
||||
Follow these instructions to build a docker container from the litellm pip package. If your company has a strict requirement around security / building images you can follow these steps.
|
||||
|
||||
**Note:** You'll need to copy the `schema.prisma` file from the [litellm repository](https://github.com/BerriAI/litellm/blob/main/schema.prisma) to your build directory alongside the Dockerfile and requirements.txt.
|
||||
|
||||
Dockerfile
|
||||
|
||||
```shell
|
||||
|
|
@ -149,6 +151,12 @@ COPY requirements.txt .
|
|||
RUN --mount=type=cache,target=${HOME}/.cache/pip \
|
||||
${HOME}/venv/bin/pip install -r requirements.txt
|
||||
|
||||
# Copy Prisma schema file
|
||||
COPY schema.prisma .
|
||||
|
||||
# Generate prisma client
|
||||
RUN prisma generate
|
||||
|
||||
EXPOSE 4000/tcp
|
||||
|
||||
ENTRYPOINT ["litellm"]
|
||||
|
|
|
|||
|
|
@ -803,11 +803,18 @@ LiteLLM Proxy supports session management for non-OpenAI models. This allows you
|
|||
|
||||
1. Enable storing request / response content in the database
|
||||
|
||||
Set `store_prompts_in_spend_logs: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the database.
|
||||
Set `store_prompts_in_cold_storage: true` in your proxy config.yaml. When this is enabled, LiteLLM will store the request and response content in the s3 bucket you specify.
|
||||
|
||||
```yaml
|
||||
litellm_settings:
|
||||
callbacks: ["s3_v2"]
|
||||
s3_callback_params: # learn more https://docs.litellm.ai/docs/proxy/logging#s3-buckets
|
||||
s3_bucket_name: litellm-logs # AWS Bucket Name for S3
|
||||
s3_region_name: us-west-2
|
||||
|
||||
general_settings:
|
||||
store_prompts_in_spend_logs: true
|
||||
cold_storage_custom_logger: s3_v2
|
||||
store_prompts_in_cold_storage: true
|
||||
```
|
||||
|
||||
2. Make request 1 with no `previous_response_id` (new session)
|
||||
|
|
|
|||
BIN
docs/my-website/img/release_notes/faster_caching_calls.png
Normal file
BIN
docs/my-website/img/release_notes/faster_caching_calls.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 270 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 472 KiB |
|
|
@ -1,5 +1,5 @@
|
|||
---
|
||||
title: "[PRE-RELEASE]v1.75.5-stable"
|
||||
title: "v1.75.5-stable - Redis latency improvements"
|
||||
slug: "v1-75-5"
|
||||
date: 2025-08-10T10:00:00
|
||||
authors:
|
||||
|
|
@ -43,8 +43,49 @@ pip install litellm==1.75.5.post1
|
|||
|
||||
---
|
||||
|
||||
## Key Highlights
|
||||
|
||||
- **Redis - Latency Improvements** - Reduces P99 latency by 50% with Redis enabled.
|
||||
- **Responses API Session Management** - Support for managing responses API sessions with images.
|
||||
- **Oracle Cloud Infrastructure** - New LLM provider for calling models on Oracle Cloud Infrastructure.
|
||||
- **Digital Ocean's Gradient AI** - New LLM provider for calling models on Digital Ocean's Gradient AI platform.
|
||||
|
||||
|
||||
### Risk of Upgrade
|
||||
|
||||
If you build the proxy from the pip package, you should hold off on upgrading. This version makes `prisma migrate deploy` our default for managing the DB. This is safer, as it doesn't reset the DB, but it requires a manual `prisma generate` step.
|
||||
|
||||
Users of our Docker image, are **not** affected by this change.
|
||||
|
||||
---
|
||||
|
||||
## Redis Latency Improvements
|
||||
|
||||
<Image
|
||||
img={require('../../img/release_notes/faster_caching_calls.png')}
|
||||
style={{width: '100%', display: 'block', margin: '2rem auto'}}
|
||||
/>
|
||||
|
||||
<br/>
|
||||
|
||||
This release adds in-memory caching for Redis requests, enabling faster response times in high-traffic. Now, LiteLLM instances will check their in-memory cache for a cache hit, before checking Redis. This reduces caching-related latency from 100ms for LLM API calls to sub-1ms, on cache hits.
|
||||
|
||||
---
|
||||
|
||||
## Responses API Session Management w/ Images
|
||||
|
||||
<Image
|
||||
img={require('../../img/release_notes/responses_api_session_mgt_images.jpg')}
|
||||
style={{width: '100%', display: 'block', margin: '2rem auto'}}
|
||||
/>
|
||||
|
||||
<br/>
|
||||
|
||||
LiteLLM now supports session management for Responses API requests with images. This is great for use-cases like chatbots, that are using the Responses API to track the state of a conversation. LiteLLM session management works across **ALL** LLM API's (including Anthropic, Bedrock, OpenAI, etc). LiteLLM session management works by storing the request and response content in an s3 bucket, you can specify.
|
||||
|
||||
---
|
||||
|
||||
|
||||
## New Models / Updated Models
|
||||
|
||||
#### New Model Support
|
||||
|
|
|
|||
231
docs/my-website/release_notes/v1.75.8/index.md
Normal file
231
docs/my-website/release_notes/v1.75.8/index.md
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
---
|
||||
title: "[PRE-RELEASE]v1.75.8"
|
||||
slug: "v1-75-8"
|
||||
date: 2025-08-16T10:00:00
|
||||
authors:
|
||||
- name: Krrish Dholakia
|
||||
title: CEO, LiteLLM
|
||||
url: https://www.linkedin.com/in/krish-d/
|
||||
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
|
||||
- name: Ishaan Jaffer
|
||||
title: CTO, LiteLLM
|
||||
url: https://www.linkedin.com/in/reffajnaahsi/
|
||||
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
|
||||
|
||||
hide_table_of_contents: false
|
||||
---
|
||||
|
||||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
## Deploy this version
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="docker" label="Docker">
|
||||
|
||||
``` showLineNumbers title="docker run litellm"
|
||||
docker run \
|
||||
-e STORE_MODEL_IN_DB=True \
|
||||
-p 4000:4000 \
|
||||
ghcr.io/berriai/litellm:v1.75.8
|
||||
```
|
||||
</TabItem>
|
||||
|
||||
<TabItem value="pip" label="Pip">
|
||||
|
||||
``` showLineNumbers title="pip install litellm"
|
||||
pip install litellm==1.75.8
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
---
|
||||
|
||||
## Key Highlights
|
||||
|
||||
- **Team Member Rate Limits** - Individual rate limiting for team members with JWT authentication support.
|
||||
- **Performance Improvements** - New experimental HTTP handler flag for 100+ RPS improvement on OpenAI calls.
|
||||
- **GPT-5 Model Family Support** - Full support for OpenAI's GPT-5 models with `reasoning_effort` parameter and Azure OpenAI integration.
|
||||
- **Azure AI Flux Image Generation** - Support for Azure AI's Flux image generation models.
|
||||
|
||||
---
|
||||
|
||||
## New Models / Updated Models
|
||||
|
||||
#### New Model Support
|
||||
|
||||
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
|
||||
| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------- |
|
||||
| Azure AI | `azure_ai/FLUX-1.1-pro` | - | - | $40/image | Image generation |
|
||||
| Azure AI | `azure_ai/FLUX.1-Kontext-pro` | - | - | $40/image | Image generation |
|
||||
| Vertex AI | `vertex_ai/deepseek-ai/deepseek-r1-0528-maas` | 65k | $1.35 | $5.4 | Chat completions + reasoning |
|
||||
| OpenRouter | `openrouter/deepseek/deepseek-chat-v3-0324` | 65k | $0.14 | $0.28 | Chat completions |
|
||||
|
||||
|
||||
#### Features
|
||||
|
||||
- **[OpenAI](../../docs/providers/openai)**
|
||||
- Added `reasoning_effort` parameter support for GPT-5 model family - [PR #13475](https://github.com/BerriAI/litellm/pull/13475), [Get Started](../../docs/providers/openai#openai-chat-completion-models)
|
||||
- Support for `reasoning` parameter in Responses API - [PR #13475](https://github.com/BerriAI/litellm/pull/13475), [Get Started](../../docs/response_api)
|
||||
- **[Azure OpenAI](../../docs/providers/azure/azure)**
|
||||
- GPT-5 support with max_tokens and `reasoning` parameter - [PR #13510](https://github.com/BerriAI/litellm/pull/13510), [Get Started](../../docs/providers/azure/azure#gpt-5-models)
|
||||
- **[AWS Bedrock](../../docs/providers/bedrock)**
|
||||
- Streaming support for bedrock gpt-oss model family - [PR #13346](https://github.com/BerriAI/litellm/pull/13346), [Get Started](../../docs/providers/bedrock#openai-gpt-oss)
|
||||
- `/messages` endpoint compatibility with `bedrock/converse/<model>` - [PR #13627](https://github.com/BerriAI/litellm/pull/13627)
|
||||
- Cache point support for assistant and tool messages - [PR #13640](https://github.com/BerriAI/litellm/pull/13640)
|
||||
- **[Azure AI](../../docs/providers/azure)**
|
||||
- New Azure AI Flux Image Generation provider - [PR #13592](https://github.com/BerriAI/litellm/pull/13592), [Get Started](../../docs/providers/azure_ai_img)
|
||||
- Fixed Content-Type header for image generation - [PR #13584](https://github.com/BerriAI/litellm/pull/13584)
|
||||
- **[CometAPI](../../docs/providers/comet)**
|
||||
- New provider support with chat completions and streaming - [PR #13458](https://github.com/BerriAI/litellm/pull/13458)
|
||||
- **[SambaNova](../../docs/providers/sambanova)**
|
||||
- Added embedding model support - [PR #13308](https://github.com/BerriAI/litellm/pull/13308), [Get Started](../../docs/providers/sambanova#sambanova---embeddings)
|
||||
- **[Vertex AI](../../docs/providers/vertex)**
|
||||
- Added `/countTokens` endpoint support for Gemini CLI integration - [PR #13545](https://github.com/BerriAI/litellm/pull/13545)
|
||||
- Token counter support for VertexAI models - [PR #13558](https://github.com/BerriAI/litellm/pull/13558)
|
||||
- **[hosted_vllm](../../docs/providers/vllm)**
|
||||
- Added `reasoning_effort` parameter support - [PR #13620](https://github.com/BerriAI/litellm/pull/13620), [Get Started](../../docs/providers/vllm#reasoning-effort)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- **[OCI](../../docs/providers/oci)**
|
||||
- Fixed streaming issues - [PR #13437](https://github.com/BerriAI/litellm/pull/13437)
|
||||
- **[Ollama](../../docs/providers/ollama)**
|
||||
- Fixed GPT-OSS streaming with 'thinking' field - [PR #13375](https://github.com/BerriAI/litellm/pull/13375)
|
||||
- **[VolcEngine](../../docs/providers/volcengine)**
|
||||
- Fixed thinking disabled parameter handling - [PR #13598](https://github.com/BerriAI/litellm/pull/13598)
|
||||
- **[Streaming](../../docs/completion/stream)**
|
||||
- Consistent 'finish_reason' chunk indexing - [PR #13560](https://github.com/BerriAI/litellm/pull/13560)
|
||||
---
|
||||
|
||||
## LLM API Endpoints
|
||||
|
||||
#### Features
|
||||
|
||||
- **[/messages](../../docs/anthropic/messages)**
|
||||
- Tool use arguments properly returned for non-anthropic models - [PR #13638](https://github.com/BerriAI/litellm/pull/13638)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- **[Real-time API](../../docs/realtime)**
|
||||
- Fixed endpoint for no intent scenarios - [PR #13476](https://github.com/BerriAI/litellm/pull/13476)
|
||||
- **[Responses API](../../docs/response_api)**
|
||||
- Fixed `stream=True` + `background=True` with Responses API - [PR #13654](https://github.com/BerriAI/litellm/pull/13654)
|
||||
|
||||
---
|
||||
|
||||
## [MCP Gateway](../../docs/mcp)
|
||||
|
||||
#### Features
|
||||
|
||||
- **Access Control & Configuration**
|
||||
- Enhanced MCPServerManager with access groups and description support - [PR #13549](https://github.com/BerriAI/litellm/pull/13549)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- **Authentication**
|
||||
- Fixed MCP gateway key authentication - [PR #13630](https://github.com/BerriAI/litellm/pull/13630)
|
||||
|
||||
[Read More](../../docs/mcp)
|
||||
|
||||
---
|
||||
|
||||
## Management Endpoints / UI
|
||||
|
||||
#### Features
|
||||
|
||||
- **Team Management**
|
||||
- Team Member Rate Limits implementation - [PR #13601](https://github.com/BerriAI/litellm/pull/13601)
|
||||
- JWT authentication support for team member rate limits - [PR #13601](https://github.com/BerriAI/litellm/pull/13601)
|
||||
- Show team member TPM/RPM limits in UI - [PR #13662](https://github.com/BerriAI/litellm/pull/13662)
|
||||
- Allow editing team member RPM/TPM limits - [PR #13669](https://github.com/BerriAI/litellm/pull/13669)
|
||||
- Allow unsetting TPM and RPM in Teams Settings - [PR #13430](https://github.com/BerriAI/litellm/pull/13430)
|
||||
- Team Member Permissions Page access column changes - [PR #13145](https://github.com/BerriAI/litellm/pull/13145)
|
||||
- **Key Management**
|
||||
- Display errors from backend on the UI Keys page - [PR #13435](https://github.com/BerriAI/litellm/pull/13435)
|
||||
- Added confirmation modal before deleting keys - [PR #13655](https://github.com/BerriAI/litellm/pull/13655)
|
||||
- Support for `user` parameter in LiteLLM SDK to Proxy communication - [PR #13555](https://github.com/BerriAI/litellm/pull/13555)
|
||||
- **UI Improvements**
|
||||
- Fixed internal users table overflow - [PR #12736](https://github.com/BerriAI/litellm/pull/12736)
|
||||
- Enhanced chart readability with short-form notation for large numbers - [PR #12370](https://github.com/BerriAI/litellm/pull/12370)
|
||||
- Fixed image overflow in LiteLLM model display - [PR #13639](https://github.com/BerriAI/litellm/pull/13639)
|
||||
- Removed ambiguous network response errors - [PR #13582](https://github.com/BerriAI/litellm/pull/13582)
|
||||
- **Credentials**
|
||||
- Added CredentialDeleteModal component and integration with CredentialsPanel - [PR #13550](https://github.com/BerriAI/litellm/pull/13550)
|
||||
- **Admin & Permissions**
|
||||
- Allow routes for admin viewer - [PR #13588](https://github.com/BerriAI/litellm/pull/13588)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- **SCIM Integration**
|
||||
- Fixed SCIM Team Memberships metadata handling - [PR #13553](https://github.com/BerriAI/litellm/pull/13553)
|
||||
- **Authentication**
|
||||
- Fixed incorrect key info endpoint - [PR #13633](https://github.com/BerriAI/litellm/pull/13633)
|
||||
|
||||
---
|
||||
|
||||
## Logging / Guardrail Integrations
|
||||
|
||||
#### Features
|
||||
|
||||
- **[Langfuse OTEL](../../docs/proxy/logging#langfuse)**
|
||||
- Added key/team logging for Langfuse OTEL Logger - [PR #13512](https://github.com/BerriAI/litellm/pull/13512)
|
||||
- Fixed LangfuseOtelSpanAttributes constants to match expected values - [PR #13659](https://github.com/BerriAI/litellm/pull/13659)
|
||||
- **[MLflow](../../docs/proxy/logging#mlflow)**
|
||||
- Updated MLflow logger usage span attributes - [PR #13561](https://github.com/BerriAI/litellm/pull/13561)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- **Security**
|
||||
- Hide sensitive data in `/model/info` - azure entra client_secret - [PR #13577](https://github.com/BerriAI/litellm/pull/13577)
|
||||
- Fixed trivy/secrets false positives - [PR #13631](https://github.com/BerriAI/litellm/pull/13631)
|
||||
|
||||
---
|
||||
|
||||
## Performance / Loadbalancing / Reliability improvements
|
||||
|
||||
#### Features
|
||||
|
||||
- **HTTP Performance**
|
||||
- New 'EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER' flag for +100 RPS improvement on OpenAI calls - [PR #13625](https://github.com/BerriAI/litellm/pull/13625)
|
||||
- **Database Monitoring**
|
||||
- Added DB metrics to Prometheus - [PR #13626](https://github.com/BerriAI/litellm/pull/13626)
|
||||
- **Error Handling**
|
||||
- Added safe divide by 0 protection to prevent crashes - [PR #13624](https://github.com/BerriAI/litellm/pull/13624)
|
||||
|
||||
#### Bugs
|
||||
|
||||
- **Dependencies**
|
||||
- Updated boto3 to 1.36.0 and aioboto3 to 13.4.0 - [PR #13665](https://github.com/BerriAI/litellm/pull/13665)
|
||||
|
||||
---
|
||||
|
||||
## General Proxy Improvements
|
||||
|
||||
#### Features
|
||||
|
||||
- **Database**
|
||||
- Removed redundant `use_prisma_migrate` flag - now default - [PR #13555](https://github.com/BerriAI/litellm/pull/13555)
|
||||
- **LLM Translation**
|
||||
- Added model ID check - [PR #13507](https://github.com/BerriAI/litellm/pull/13507)
|
||||
- Refactored Anthropic configurations and added support for `anthropic_beta` headers - [PR #13590](https://github.com/BerriAI/litellm/pull/13590)
|
||||
|
||||
|
||||
---
|
||||
|
||||
## New Contributors
|
||||
* @TensorNull made their first contribution in [PR #13458](https://github.com/BerriAI/litellm/pull/13458)
|
||||
* @MajorD00m made their first contribution in [PR #13577](https://github.com/BerriAI/litellm/pull/13577)
|
||||
* @VerunicaM made their first contribution in [PR #13584](https://github.com/BerriAI/litellm/pull/13584)
|
||||
* @huangyafei made their first contribution in [PR #13607](https://github.com/BerriAI/litellm/pull/13607)
|
||||
* @TomeHirata made their first contribution in [PR #13561](https://github.com/BerriAI/litellm/pull/13561)
|
||||
* @willfinnigan made their first contribution in [PR #13659](https://github.com/BerriAI/litellm/pull/13659)
|
||||
* @dcbark01 made their first contribution in [PR #13633](https://github.com/BerriAI/litellm/pull/13633)
|
||||
* @javacruft made their first contribution in [PR #13631](https://github.com/BerriAI/litellm/pull/13631)
|
||||
|
||||
---
|
||||
|
||||
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.75.5-stable.rc-draft...v1.75.8-nightly)**
|
||||
|
||||
|
|
@ -372,7 +372,14 @@ const sidebars = {
|
|||
"providers/azure/azure_embedding",
|
||||
]
|
||||
},
|
||||
"providers/azure_ai",
|
||||
{
|
||||
type: "category",
|
||||
label: "Azure AI",
|
||||
items: [
|
||||
"providers/azure_ai",
|
||||
"providers/azure_ai_img",
|
||||
]
|
||||
},
|
||||
{
|
||||
type: "category",
|
||||
label: "Vertex AI",
|
||||
|
|
|
|||
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.17-py3-none-any.whl
vendored
Normal file
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.17-py3-none-any.whl
vendored
Normal file
Binary file not shown.
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.17.tar.gz
vendored
Normal file
BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.17.tar.gz
vendored
Normal file
Binary file not shown.
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.2.16"
|
||||
version = "0.2.17"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
authors = ["BerriAI"]
|
||||
readme = "README.md"
|
||||
|
|
@ -22,7 +22,7 @@ requires = ["poetry-core"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.2.16"
|
||||
version = "0.2.17"
|
||||
version_files = [
|
||||
"pyproject.toml:version",
|
||||
"../requirements.txt:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -486,6 +486,7 @@ vertex_code_text_models: List = []
|
|||
vertex_embedding_models: List = []
|
||||
vertex_anthropic_models: List = []
|
||||
vertex_llama3_models: List = []
|
||||
vertex_deepseek_models: List = []
|
||||
vertex_ai_ai21_models: List = []
|
||||
vertex_mistral_models: List = []
|
||||
ai21_models: List = []
|
||||
|
|
@ -618,6 +619,9 @@ def add_known_models():
|
|||
elif value.get("litellm_provider") == "vertex_ai-llama_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_llama3_models.append(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-deepseek_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_deepseek_models.append(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-mistral_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_mistral_models.append(key)
|
||||
|
|
@ -850,7 +854,8 @@ models_by_provider: dict = {
|
|||
+ vertex_text_models
|
||||
+ vertex_anthropic_models
|
||||
+ vertex_vision_models
|
||||
+ vertex_language_models,
|
||||
+ vertex_language_models
|
||||
+ vertex_deepseek_models,
|
||||
"ai21": ai21_models,
|
||||
"bedrock": bedrock_models + bedrock_converse_models,
|
||||
"petals": petals_models,
|
||||
|
|
@ -1041,7 +1046,7 @@ from .llms.anthropic.experimental_pass_through.messages.transformation import (
|
|||
AnthropicMessagesConfig,
|
||||
)
|
||||
from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
|
||||
AmazonAnthropicClaude3MessagesConfig,
|
||||
AmazonAnthropicClaudeMessagesConfig,
|
||||
)
|
||||
from .llms.together_ai.chat import TogetherAIConfig
|
||||
from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig
|
||||
|
|
@ -1101,7 +1106,7 @@ from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation
|
|||
AmazonAnthropicConfig,
|
||||
)
|
||||
from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
||||
AmazonAnthropicClaude3Config,
|
||||
AmazonAnthropicClaudeConfig,
|
||||
)
|
||||
from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import (
|
||||
AmazonCohereConfig,
|
||||
|
|
|
|||
|
|
@ -32,9 +32,6 @@ from litellm.llms.azure.cost_calculation import (
|
|||
from litellm.llms.bedrock.cost_calculation import (
|
||||
cost_per_token as bedrock_cost_per_token,
|
||||
)
|
||||
from litellm.llms.bedrock.image.cost_calculator import (
|
||||
cost_calculator as bedrock_image_cost_calculator,
|
||||
)
|
||||
from litellm.llms.databricks.cost_calculator import (
|
||||
cost_per_token as databricks_cost_per_token,
|
||||
)
|
||||
|
|
@ -60,9 +57,6 @@ from litellm.llms.vertex_ai.cost_calculator import (
|
|||
cost_per_token as google_cost_per_token,
|
||||
)
|
||||
from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_router
|
||||
from litellm.llms.vertex_ai.image_generation.cost_calculator import (
|
||||
cost_calculator as vertex_ai_image_cost_calculator,
|
||||
)
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.types.llms.openai import (
|
||||
HttpxBinaryResponseContent,
|
||||
|
|
@ -768,50 +762,15 @@ def completion_cost( # noqa: PLR0915
|
|||
)
|
||||
if CostCalculatorUtils._call_type_has_image_response(call_type):
|
||||
### IMAGE GENERATION COST CALCULATION ###
|
||||
if custom_llm_provider == "vertex_ai":
|
||||
if isinstance(completion_response, ImageResponse):
|
||||
return vertex_ai_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
)
|
||||
elif custom_llm_provider == "bedrock":
|
||||
if isinstance(completion_response, ImageResponse):
|
||||
return bedrock_image_cost_calculator(
|
||||
model=model,
|
||||
size=size,
|
||||
image_response=completion_response,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
raise TypeError(
|
||||
"completion_response must be of type ImageResponse for bedrock image cost calculation"
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
|
||||
from litellm.llms.recraft.cost_calculator import (
|
||||
cost_calculator as recraft_image_cost_calculator,
|
||||
)
|
||||
|
||||
return recraft_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
|
||||
from litellm.llms.gemini.image_generation.cost_calculator import (
|
||||
cost_calculator as gemini_image_cost_calculator,
|
||||
)
|
||||
|
||||
return gemini_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
)
|
||||
else:
|
||||
return default_image_cost_calculator(
|
||||
model=model,
|
||||
quality=quality,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
n=n,
|
||||
size=size,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
return CostCalculatorUtils.route_image_generation_cost_calculator(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
completion_response=completion_response,
|
||||
quality=quality,
|
||||
n=n,
|
||||
size=size,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
elif (
|
||||
call_type == CallTypes.speech.value
|
||||
or call_type == CallTypes.aspeech.value
|
||||
|
|
|
|||
|
|
@ -311,7 +311,7 @@ def image_generation( # noqa: PLR0915
|
|||
) or get_secret_str("AZURE_AD_TOKEN")
|
||||
|
||||
default_headers = {
|
||||
"Content-Type": "application/json;",
|
||||
"Content-Type": "application/json",
|
||||
"api-key": api_key,
|
||||
}
|
||||
for k, v in default_headers.items():
|
||||
|
|
@ -335,6 +335,38 @@ def image_generation( # noqa: PLR0915
|
|||
headers=headers,
|
||||
litellm_params=litellm_params_dict,
|
||||
)
|
||||
elif custom_llm_provider == "azure_ai":
|
||||
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
|
||||
api_base = AzureFoundryModelInfo.get_api_base(api_base)
|
||||
api_key = AzureFoundryModelInfo.get_api_key(api_key)
|
||||
if extra_headers is not None:
|
||||
optional_params["extra_headers"] = extra_headers
|
||||
|
||||
default_headers = {
|
||||
"Content-Type": "application/json",
|
||||
"api-key": api_key,
|
||||
}
|
||||
for k, v in default_headers.items():
|
||||
if k not in headers:
|
||||
headers[k] = v
|
||||
|
||||
model_response = azure_chat_completions.image_generation(
|
||||
model=model,
|
||||
prompt=prompt,
|
||||
timeout=timeout,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
azure_ad_token=None,
|
||||
azure_ad_token_provider=azure_ad_token_provider,
|
||||
logging_obj=litellm_logging_obj,
|
||||
optional_params=optional_params,
|
||||
model_response=model_response,
|
||||
api_version=api_version,
|
||||
aimg_generation=aimg_generation,
|
||||
client=client,
|
||||
headers=headers,
|
||||
litellm_params=litellm_params_dict,
|
||||
)
|
||||
elif (
|
||||
custom_llm_provider == "openai"
|
||||
or custom_llm_provider in litellm.openai_compatible_providers
|
||||
|
|
|
|||
|
|
@ -189,9 +189,9 @@ class MlflowLogger(CustomLogger):
|
|||
{
|
||||
"api_base": standard_obj.get("api_base"),
|
||||
"cache_hit": standard_obj.get("cache_hit"),
|
||||
"usage": {
|
||||
"completion_tokens": standard_obj.get("completion_tokens"),
|
||||
"prompt_tokens": standard_obj.get("prompt_tokens"),
|
||||
"mlflow.chat.tokenUsage": {
|
||||
"input_tokens": standard_obj.get("prompt_tokens"),
|
||||
"output_tokens": standard_obj.get("completion_tokens"),
|
||||
"total_tokens": standard_obj.get("total_tokens"),
|
||||
},
|
||||
"raw_llm_response": standard_obj.get("response"),
|
||||
|
|
|
|||
|
|
@ -37,6 +37,27 @@ def safe_divide_seconds(
|
|||
return float(seconds / denominator)
|
||||
|
||||
|
||||
def safe_divide(
|
||||
numerator: Union[int, float],
|
||||
denominator: Union[int, float],
|
||||
default: Union[int, float] = 0
|
||||
) -> Union[int, float]:
|
||||
"""
|
||||
Safely divide two numbers, returning a default value if denominator is zero.
|
||||
|
||||
Args:
|
||||
numerator: The number to divide
|
||||
denominator: The number to divide by
|
||||
default: Value to return if denominator is zero (defaults to 0)
|
||||
|
||||
Returns:
|
||||
The result of numerator/denominator, or default if denominator is zero
|
||||
"""
|
||||
if denominator == 0:
|
||||
return default
|
||||
return numerator / denominator
|
||||
|
||||
|
||||
def map_finish_reason(
|
||||
finish_reason: str,
|
||||
): # openai supports 5 stop sequences - 'stop', 'length', 'function_call', 'content_filter', 'null'
|
||||
|
|
|
|||
|
|
@ -811,7 +811,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
str(e)
|
||||
)
|
||||
)
|
||||
if self.logger_fn and callable(self.logger_fn):
|
||||
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
|
||||
try:
|
||||
self.logger_fn(
|
||||
self.model_call_details
|
||||
|
|
@ -999,7 +999,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
)
|
||||
)
|
||||
)
|
||||
if self.logger_fn and callable(self.logger_fn):
|
||||
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
|
||||
try:
|
||||
self.logger_fn(
|
||||
self.model_call_details
|
||||
|
|
@ -3919,10 +3919,12 @@ class StandardLoggingPayloadSetup:
|
|||
|
||||
# Generate cold storage object key if cold storage is configured
|
||||
if start_time is not None and response_id is not None:
|
||||
cold_storage_object_key = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
|
||||
start_time=start_time,
|
||||
response_id=response_id,
|
||||
team_alias=clean_metadata.get("user_api_key_team_alias"),
|
||||
cold_storage_object_key = (
|
||||
StandardLoggingPayloadSetup._generate_cold_storage_object_key(
|
||||
start_time=start_time,
|
||||
response_id=response_id,
|
||||
team_alias=clean_metadata.get("user_api_key_team_alias"),
|
||||
)
|
||||
)
|
||||
if cold_storage_object_key:
|
||||
clean_metadata["cold_storage_object_key"] = cold_storage_object_key
|
||||
|
|
@ -4093,12 +4095,12 @@ class StandardLoggingPayloadSetup:
|
|||
) -> Optional[str]:
|
||||
"""
|
||||
Generate cold storage object key in the same format as S3Logger.
|
||||
|
||||
|
||||
Args:
|
||||
start_time: The start time of the request
|
||||
response_id: The response ID
|
||||
response_id: The response ID
|
||||
team_alias: Optional team alias for team-based prefixing
|
||||
|
||||
|
||||
Returns:
|
||||
Optional[str]: The generated object key or None if cold storage not configured
|
||||
"""
|
||||
|
|
@ -4112,26 +4114,23 @@ class StandardLoggingPayloadSetup:
|
|||
ColdStorageHandler._get_configured_cold_storage_custom_logger()
|
||||
)
|
||||
except Exception as e:
|
||||
verbose_logger.debug(
|
||||
f"Cold storage custom logger unavailable: {e}"
|
||||
)
|
||||
verbose_logger.debug(f"Cold storage custom logger unavailable: {e}")
|
||||
return None
|
||||
|
||||
if configured_cold_storage_logger is None:
|
||||
return None
|
||||
|
||||
|
||||
try:
|
||||
# Generate file name in same format as litellm.utils.get_logging_id
|
||||
s3_file_name = f"time-{start_time.strftime('%H-%M-%S-%f')}_{response_id}"
|
||||
|
||||
|
||||
s3_object_key = get_s3_object_key(
|
||||
s3_path="", # Use empty path as default
|
||||
team_alias_prefix="", # Don't split by team alias for cold storage
|
||||
s3_path="", # Use empty path as default
|
||||
team_alias_prefix="", # Don't split by team alias for cold storage
|
||||
start_time=start_time,
|
||||
s3_file_name=s3_file_name,
|
||||
)
|
||||
|
||||
|
||||
return s3_object_key
|
||||
except Exception:
|
||||
# If any error occurs in generating the key, return None
|
||||
|
|
|
|||
|
|
@ -1,11 +1,17 @@
|
|||
# What is this?
|
||||
## Helper utilities for cost_per_token()
|
||||
|
||||
from typing import Literal, Optional, Tuple, cast
|
||||
from typing import Any, Literal, Optional, Tuple, cast
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.types.utils import CallTypes, ModelInfo, PassthroughCallTypes, Usage
|
||||
from litellm.types.utils import (
|
||||
CallTypes,
|
||||
ImageResponse,
|
||||
ModelInfo,
|
||||
PassthroughCallTypes,
|
||||
Usage,
|
||||
)
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
||||
|
|
@ -377,3 +383,84 @@ class CostCalculatorUtils:
|
|||
]:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def route_image_generation_cost_calculator(
|
||||
model: str,
|
||||
completion_response: Any,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
quality: Optional[str] = None,
|
||||
n: Optional[int] = None,
|
||||
size: Optional[str] = None,
|
||||
optional_params: Optional[dict] = None,
|
||||
) -> float:
|
||||
"""
|
||||
Route the image generation cost calculator based on the custom_llm_provider
|
||||
"""
|
||||
from litellm.cost_calculator import default_image_cost_calculator
|
||||
from litellm.llms.azure_ai.image_generation.cost_calculator import (
|
||||
cost_calculator as azure_ai_image_cost_calculator,
|
||||
)
|
||||
from litellm.llms.bedrock.image.cost_calculator import (
|
||||
cost_calculator as bedrock_image_cost_calculator,
|
||||
)
|
||||
from litellm.llms.gemini.image_generation.cost_calculator import (
|
||||
cost_calculator as gemini_image_cost_calculator,
|
||||
)
|
||||
from litellm.llms.recraft.cost_calculator import (
|
||||
cost_calculator as recraft_image_cost_calculator,
|
||||
)
|
||||
from litellm.llms.vertex_ai.image_generation.cost_calculator import (
|
||||
cost_calculator as vertex_ai_image_cost_calculator,
|
||||
)
|
||||
|
||||
if custom_llm_provider == litellm.LlmProviders.VERTEX_AI.value:
|
||||
if isinstance(completion_response, ImageResponse):
|
||||
return vertex_ai_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.BEDROCK.value:
|
||||
if isinstance(completion_response, ImageResponse):
|
||||
return bedrock_image_cost_calculator(
|
||||
model=model,
|
||||
size=size,
|
||||
image_response=completion_response,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
raise TypeError(
|
||||
"completion_response must be of type ImageResponse for bedrock image cost calculation"
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.RECRAFT.value:
|
||||
from litellm.llms.recraft.cost_calculator import (
|
||||
cost_calculator as recraft_image_cost_calculator,
|
||||
)
|
||||
|
||||
return recraft_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.GEMINI.value:
|
||||
from litellm.llms.gemini.image_generation.cost_calculator import (
|
||||
cost_calculator as gemini_image_cost_calculator,
|
||||
)
|
||||
|
||||
return gemini_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
)
|
||||
elif custom_llm_provider == litellm.LlmProviders.AZURE_AI.value:
|
||||
return azure_ai_image_cost_calculator(
|
||||
model=model,
|
||||
image_response=completion_response,
|
||||
)
|
||||
else:
|
||||
return default_image_cost_calculator(
|
||||
model=model,
|
||||
quality=quality,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
n=n,
|
||||
size=size,
|
||||
optional_params=optional_params,
|
||||
)
|
||||
return 0.0
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_
|
|||
from litellm.types.files import get_file_extension_from_mime_type
|
||||
from litellm.types.llms.anthropic import *
|
||||
from litellm.types.llms.bedrock import MessageBlock as BedrockMessageBlock
|
||||
from litellm.types.llms.bedrock import CachePointBlock
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
from litellm.types.llms.ollama import OllamaVisionModelObject
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -2685,6 +2686,11 @@ def _convert_to_bedrock_tool_call_invoke(
|
|||
)
|
||||
bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool)
|
||||
_parts_list.append(bedrock_content_block)
|
||||
|
||||
# Check for cache_control and add a separate cachePoint block
|
||||
if tool.get("cache_control", None) is not None:
|
||||
cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
|
||||
_parts_list.append(cache_point_block)
|
||||
return _parts_list
|
||||
except Exception as e:
|
||||
raise Exception(
|
||||
|
|
@ -2745,6 +2751,7 @@ def _convert_to_bedrock_tool_call_result(
|
|||
for content in content_list:
|
||||
if content["type"] == "text":
|
||||
content_str += content["text"]
|
||||
|
||||
message.get("name", "")
|
||||
id = str(message.get("tool_call_id", str(uuid.uuid4())))
|
||||
|
||||
|
|
@ -2753,6 +2760,7 @@ def _convert_to_bedrock_tool_call_result(
|
|||
content=[tool_result_content_block],
|
||||
toolUseId=id,
|
||||
)
|
||||
|
||||
content_block = BedrockContentBlock(toolResult=tool_result)
|
||||
|
||||
return content_block
|
||||
|
|
@ -3516,8 +3524,30 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
|
|||
tool_content: List[BedrockContentBlock] = []
|
||||
while msg_i < len(messages) and messages[msg_i]["role"] == "tool":
|
||||
tool_call_result = _convert_to_bedrock_tool_call_result(messages[msg_i])
|
||||
|
||||
current_message = messages[msg_i]
|
||||
|
||||
# Add the tool result first
|
||||
tool_content.append(tool_call_result)
|
||||
|
||||
# Check if we need to add a separate cachePoint block
|
||||
has_cache_control = False
|
||||
|
||||
# Check for message-level cache_control
|
||||
if current_message.get("cache_control", None) is not None:
|
||||
has_cache_control = True
|
||||
# Check for content-level cache_control in list content
|
||||
elif isinstance(current_message.get("content"), list):
|
||||
for content_element in current_message["content"]:
|
||||
if (isinstance(content_element, dict) and
|
||||
content_element.get("cache_control", None) is not None):
|
||||
has_cache_control = True
|
||||
break
|
||||
|
||||
# Add a separate cachePoint block if cache_control is present
|
||||
if has_cache_control:
|
||||
cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
|
||||
tool_content.append(cache_point_block)
|
||||
|
||||
msg_i += 1
|
||||
if tool_content:
|
||||
# if last message was a 'user' message, then add a blank assistant message (bedrock requires alternating roles)
|
||||
|
|
@ -3589,9 +3619,28 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
|
|||
image_url=image_url
|
||||
)
|
||||
assistants_parts.append(assistants_part)
|
||||
# Add cache point block for assistant content elements
|
||||
_cache_point_block = (
|
||||
litellm.AmazonConverseConfig()._get_cache_point_block(
|
||||
message_block=cast(
|
||||
OpenAIMessageContentListBlock, element
|
||||
),
|
||||
block_type="content_block",
|
||||
)
|
||||
)
|
||||
if _cache_point_block is not None:
|
||||
assistants_parts.append(_cache_point_block)
|
||||
assistant_content.extend(assistants_parts)
|
||||
elif _assistant_content is not None and isinstance(_assistant_content, str):
|
||||
assistant_content.append(BedrockContentBlock(text=_assistant_content))
|
||||
# Add cache point block for assistant string content
|
||||
_cache_point_block = (
|
||||
litellm.AmazonConverseConfig()._get_cache_point_block(
|
||||
assistant_message_block, block_type="content_block"
|
||||
)
|
||||
)
|
||||
if _cache_point_block is not None:
|
||||
assistant_content.append(_cache_point_block)
|
||||
_tool_calls = assistant_message_block.get("tool_calls", [])
|
||||
if _tool_calls:
|
||||
assistant_content.extend(
|
||||
|
|
|
|||
|
|
@ -1747,6 +1747,11 @@ class CustomStreamWrapper:
|
|||
if is_empty:
|
||||
continue
|
||||
print_verbose(f"final returned processed chunk: {processed_chunk}")
|
||||
|
||||
# add usage as hidden param
|
||||
if self.sent_last_chunk is True and self.stream_options is None:
|
||||
usage = calculate_total_usage(chunks=self.chunks)
|
||||
processed_chunk._hidden_params["usage"] = usage
|
||||
return processed_chunk
|
||||
raise StopAsyncIteration
|
||||
else: # temporary patch for non-aiohttp async calls
|
||||
|
|
@ -1790,6 +1795,7 @@ class CustomStreamWrapper:
|
|||
messages=self.messages,
|
||||
logging_obj=self.logging_obj,
|
||||
)
|
||||
|
||||
response = self.model_response_creator()
|
||||
if complete_streaming_response is not None:
|
||||
setattr(
|
||||
|
|
|
|||
|
|
@ -489,7 +489,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
text: str = ""
|
||||
partial_json: Optional[str] = None
|
||||
for choice in choices:
|
||||
if choice.delta.content is not None:
|
||||
if choice.delta.content is not None and len(choice.delta.content) > 0:
|
||||
text += choice.delta.content
|
||||
elif choice.delta.tool_calls is not None:
|
||||
partial_json = ""
|
||||
|
|
@ -499,7 +499,6 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
and tool.function.arguments is not None
|
||||
):
|
||||
partial_json += tool.function.arguments
|
||||
|
||||
if partial_json is not None:
|
||||
return "input_json_delta", ContentJsonBlockDelta(
|
||||
type="input_json_delta", partial_json=partial_json
|
||||
|
|
|
|||
56
litellm/llms/azure_ai/common_utils.py
Normal file
56
litellm/llms/azure_ai/common_utils.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
from typing import List, Optional
|
||||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
|
||||
class AzureFoundryModelInfo(BaseLLMModelInfo):
|
||||
@staticmethod
|
||||
def get_api_base(api_base: Optional[str] = None) -> Optional[str]:
|
||||
return (
|
||||
api_base
|
||||
or litellm.api_base
|
||||
or get_secret_str("AZURE_AI_API_BASE")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
|
||||
return (
|
||||
api_key
|
||||
or litellm.api_key
|
||||
or litellm.openai_key
|
||||
or get_secret_str("AZURE_AI_API_KEY")
|
||||
)
|
||||
|
||||
@property
|
||||
def api_version(self, api_version: Optional[str] = None) -> Optional[str]:
|
||||
api_version = (
|
||||
api_version
|
||||
or litellm.api_version
|
||||
or get_secret_str("AZURE_API_VERSION")
|
||||
)
|
||||
return api_version
|
||||
|
||||
#########################################################
|
||||
# Not implemented methods
|
||||
#########################################################
|
||||
|
||||
|
||||
@staticmethod
|
||||
def get_base_model(model: str) -> Optional[str]:
|
||||
raise NotImplementedError("Azure Foundry does not support base model")
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
model: str,
|
||||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""Azure Foundry sends api key in query params"""
|
||||
raise NotImplementedError("Azure Foundry does not support environment validation")
|
||||
33
litellm/llms/azure_ai/image_generation/__init__.py
Normal file
33
litellm/llms/azure_ai/image_generation/__init__.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
)
|
||||
|
||||
from .dall_e_2_transformation import AzureFoundryDallE2ImageGenerationConfig
|
||||
from .dall_e_3_transformation import AzureFoundryDallE3ImageGenerationConfig
|
||||
from .flux_transformation import AzureFoundryFluxImageGenerationConfig
|
||||
from .gpt_transformation import AzureFoundryGPTImageGenerationConfig
|
||||
|
||||
__all__ = [
|
||||
"AzureFoundryFluxImageGenerationConfig",
|
||||
"AzureFoundryGPTImageGenerationConfig",
|
||||
"AzureFoundryDallE2ImageGenerationConfig",
|
||||
"AzureFoundryDallE3ImageGenerationConfig",
|
||||
]
|
||||
|
||||
|
||||
def get_azure_ai_image_generation_config(model: str) -> BaseImageGenerationConfig:
|
||||
model = model.lower()
|
||||
model = model.replace("-", "")
|
||||
model = model.replace("_", "")
|
||||
if model == "" or "dalle2" in model: # empty model is dall-e-2
|
||||
return AzureFoundryDallE2ImageGenerationConfig()
|
||||
elif "dalle3" in model:
|
||||
return AzureFoundryDallE3ImageGenerationConfig()
|
||||
elif "flux" in model:
|
||||
return AzureFoundryFluxImageGenerationConfig()
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image-1 model format."
|
||||
)
|
||||
return AzureFoundryGPTImageGenerationConfig()
|
||||
25
litellm/llms/azure_ai/image_generation/cost_calculator.py
Normal file
25
litellm/llms/azure_ai/image_generation/cost_calculator.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
from typing import Any
|
||||
|
||||
import litellm
|
||||
from litellm.types.utils import ImageResponse
|
||||
|
||||
|
||||
def cost_calculator(
|
||||
model: str,
|
||||
image_response: Any,
|
||||
) -> float:
|
||||
"""
|
||||
Recraft image generation cost calculator
|
||||
"""
|
||||
_model_info = litellm.get_model_info(
|
||||
model=model,
|
||||
custom_llm_provider=litellm.LlmProviders.AZURE_AI.value,
|
||||
)
|
||||
output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0
|
||||
num_images: int = 0
|
||||
if isinstance(image_response, ImageResponse):
|
||||
if image_response.data:
|
||||
num_images = len(image_response.data)
|
||||
return output_cost_per_image * num_images
|
||||
else:
|
||||
raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
from litellm.llms.openai.image_generation import DallE2ImageGenerationConfig
|
||||
|
||||
|
||||
class AzureFoundryDallE2ImageGenerationConfig(DallE2ImageGenerationConfig):
|
||||
"""
|
||||
Azure dall-e-2 image generation config
|
||||
"""
|
||||
|
||||
pass
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
from litellm.llms.openai.image_generation import DallE3ImageGenerationConfig
|
||||
|
||||
|
||||
class AzureFoundryDallE3ImageGenerationConfig(DallE3ImageGenerationConfig):
|
||||
"""
|
||||
Azure dall-e-3 image generation config
|
||||
"""
|
||||
|
||||
pass
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
from litellm.llms.openai.image_generation import GPTImageGenerationConfig
|
||||
|
||||
|
||||
class AzureFoundryFluxImageGenerationConfig(GPTImageGenerationConfig):
|
||||
"""
|
||||
Azure Foundry flux image generation config
|
||||
|
||||
From manual testing it follows the gpt-image-1 image generation config
|
||||
|
||||
(Azure Foundry does not have any docs on supported params at the time of writing)
|
||||
|
||||
From our test suite - following GPTImageGenerationConfig is working for this model
|
||||
"""
|
||||
pass
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
from litellm.llms.openai.image_generation import GPTImageGenerationConfig
|
||||
|
||||
|
||||
class AzureFoundryGPTImageGenerationConfig(GPTImageGenerationConfig):
|
||||
"""
|
||||
Azure gpt-image-1 image generation config
|
||||
"""
|
||||
|
||||
pass
|
||||
|
|
@ -119,6 +119,7 @@ class BedrockConverseLLM(BaseAWSLLM):
|
|||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
data = json.dumps(request_data)
|
||||
|
||||
|
|
@ -185,8 +186,10 @@ class BedrockConverseLLM(BaseAWSLLM):
|
|||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
data = json.dumps(request_data)
|
||||
|
||||
prepped = self.get_request_headers(
|
||||
credentials=credentials,
|
||||
aws_region_name=litellm_params.get("aws_region_name") or "us-west-2",
|
||||
|
|
@ -390,8 +393,10 @@ class BedrockConverseLLM(BaseAWSLLM):
|
|||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=extra_headers,
|
||||
)
|
||||
data = json.dumps(_data)
|
||||
|
||||
prepped = self.get_request_headers(
|
||||
credentials=credentials,
|
||||
aws_region_name=aws_region_name,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti
|
|||
from litellm.types.llms.bedrock import *
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantMessage,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
ChatCompletionResponseMessage,
|
||||
ChatCompletionSystemMessage,
|
||||
|
|
@ -47,7 +48,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
from litellm.utils import add_dummy_tool, has_tool_call_blocks, supports_reasoning
|
||||
|
||||
from ..common_utils import BedrockError, BedrockModelInfo, get_bedrock_tool_name
|
||||
from ..common_utils import BedrockError, BedrockModelInfo, get_bedrock_tool_name, get_anthropic_beta_from_headers
|
||||
|
||||
# Computer use tool prefixes supported by Bedrock
|
||||
BEDROCK_COMPUTER_USE_TOOLS = [
|
||||
|
|
@ -505,6 +506,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
OpenAIMessageContentListBlock,
|
||||
ChatCompletionUserMessage,
|
||||
ChatCompletionSystemMessage,
|
||||
ChatCompletionAssistantMessage,
|
||||
],
|
||||
block_type: Literal["system"],
|
||||
) -> Optional[SystemContentBlock]:
|
||||
|
|
@ -517,6 +519,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
OpenAIMessageContentListBlock,
|
||||
ChatCompletionUserMessage,
|
||||
ChatCompletionSystemMessage,
|
||||
ChatCompletionAssistantMessage,
|
||||
],
|
||||
block_type: Literal["content_block"],
|
||||
) -> Optional[ContentBlock]:
|
||||
|
|
@ -528,6 +531,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
OpenAIMessageContentListBlock,
|
||||
ChatCompletionUserMessage,
|
||||
ChatCompletionSystemMessage,
|
||||
ChatCompletionAssistantMessage,
|
||||
],
|
||||
block_type: Literal["system", "content_block"],
|
||||
) -> Optional[Union[SystemContentBlock, ContentBlock]]:
|
||||
|
|
@ -593,12 +597,14 @@ class AmazonConverseConfig(BaseConfig):
|
|||
|
||||
return {}
|
||||
|
||||
|
||||
def _transform_request_helper(
|
||||
self,
|
||||
model: str,
|
||||
system_content_blocks: List[SystemContentBlock],
|
||||
optional_params: dict,
|
||||
messages: Optional[List[AllMessageValues]] = None,
|
||||
headers: Optional[dict] = None,
|
||||
) -> CommonRequestObject:
|
||||
## VALIDATE REQUEST
|
||||
"""
|
||||
|
|
@ -651,6 +657,12 @@ class AmazonConverseConfig(BaseConfig):
|
|||
# Initialize bedrock_tools
|
||||
bedrock_tools: List[ToolBlock] = []
|
||||
|
||||
# Collect anthropic_beta values from user headers
|
||||
anthropic_beta_list = []
|
||||
if headers:
|
||||
user_betas = get_anthropic_beta_from_headers(headers)
|
||||
anthropic_beta_list.extend(user_betas)
|
||||
|
||||
# Only separate tools if computer use tools are actually present
|
||||
if original_tools and self.is_computer_use_tool_used(original_tools, model):
|
||||
# Separate computer use tools from regular function tools
|
||||
|
|
@ -663,7 +675,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
|
||||
# Add computer use tools and anthropic_beta if needed (only when computer use tools are present)
|
||||
if computer_use_tools:
|
||||
additional_request_params["anthropic_beta"] = ["computer-use-2024-10-22"]
|
||||
anthropic_beta_list.append("computer-use-2024-10-22")
|
||||
# Transform computer use tools to proper Bedrock format
|
||||
transformed_computer_tools = self._transform_computer_use_tools(computer_use_tools)
|
||||
additional_request_params["tools"] = transformed_computer_tools
|
||||
|
|
@ -671,6 +683,17 @@ class AmazonConverseConfig(BaseConfig):
|
|||
# No computer use tools, process all tools as regular tools
|
||||
bedrock_tools = _bedrock_tools_pt(original_tools)
|
||||
|
||||
# Set anthropic_beta in additional_request_params if we have any beta features
|
||||
if anthropic_beta_list:
|
||||
# Remove duplicates while preserving order
|
||||
unique_betas = []
|
||||
seen = set()
|
||||
for beta in anthropic_beta_list:
|
||||
if beta not in seen:
|
||||
unique_betas.append(beta)
|
||||
seen.add(beta)
|
||||
additional_request_params["anthropic_beta"] = unique_betas
|
||||
|
||||
bedrock_tool_config: Optional[ToolConfigBlock] = None
|
||||
if len(bedrock_tools) > 0:
|
||||
tool_choice_values: ToolChoiceValuesBlock = inference_params.pop(
|
||||
|
|
@ -708,6 +731,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: Optional[dict] = None,
|
||||
) -> RequestObject:
|
||||
messages, system_content_blocks = self._transform_system_message(messages)
|
||||
## TRANSFORMATION ##
|
||||
|
|
@ -717,6 +741,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
system_content_blocks=system_content_blocks,
|
||||
optional_params=optional_params,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
bedrock_messages = (
|
||||
|
|
@ -747,6 +772,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -756,6 +782,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
messages: List[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
headers: Optional[dict] = None,
|
||||
) -> RequestObject:
|
||||
messages, system_content_blocks = self._transform_system_message(messages)
|
||||
|
||||
|
|
@ -764,6 +791,7 @@ class AmazonConverseConfig(BaseConfig):
|
|||
system_content_blocks=system_content_blocks,
|
||||
optional_params=optional_params,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
## TRANSFORMATION ##
|
||||
|
|
|
|||
|
|
@ -831,7 +831,7 @@ class BedrockLLM(BaseAWSLLM):
|
|||
model=model, messages=messages, custom_llm_provider="anthropic_xml"
|
||||
) # type: ignore
|
||||
## LOAD CONFIG
|
||||
config = litellm.AmazonAnthropicClaude3Config.get_config()
|
||||
config = litellm.AmazonAnthropicClaudeConfig.get_config()
|
||||
for k, v in config.items():
|
||||
if (
|
||||
k not in inference_params
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
|||
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
||||
AmazonInvokeConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import get_anthropic_beta_from_headers
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
|
@ -17,13 +18,22 @@ else:
|
|||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
class AmazonAnthropicClaude3Config(AmazonInvokeConfig, AnthropicConfig):
|
||||
class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
|
||||
"""
|
||||
Reference:
|
||||
https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=claude
|
||||
https://docs.anthropic.com/claude/docs/models-overview#model-comparison
|
||||
https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html
|
||||
|
||||
Supported Params for the Amazon / Anthropic Claude 3 models:
|
||||
Supported Params for the Amazon / Anthropic Claude models (Claude 3, Claude 4, etc.):
|
||||
Supports anthropic_beta parameter for beta features like:
|
||||
- computer-use-2025-01-24 (Claude 3.7 Sonnet)
|
||||
- computer-use-2024-10-22 (Claude 3.5 Sonnet v2)
|
||||
- token-efficient-tools-2025-02-19 (Claude 3.7 Sonnet)
|
||||
- interleaved-thinking-2025-05-14 (Claude 4 models)
|
||||
- output-128k-2025-02-19 (Claude 3.7 Sonnet)
|
||||
- dev-full-thinking-2025-05-14 (Claude 4 models)
|
||||
- context-1m-2025-08-07 (Claude Sonnet 4)
|
||||
"""
|
||||
|
||||
anthropic_version: str = "bedrock-2023-05-31"
|
||||
|
|
@ -50,6 +60,7 @@ class AmazonAnthropicClaude3Config(AmazonInvokeConfig, AnthropicConfig):
|
|||
drop_params,
|
||||
)
|
||||
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -72,6 +83,11 @@ class AmazonAnthropicClaude3Config(AmazonInvokeConfig, AnthropicConfig):
|
|||
if "anthropic_version" not in _anthropic_request:
|
||||
_anthropic_request["anthropic_version"] = self.anthropic_version
|
||||
|
||||
# Handle anthropic_beta from user headers
|
||||
anthropic_beta_list = get_anthropic_beta_from_headers(headers)
|
||||
if anthropic_beta_list:
|
||||
_anthropic_request["anthropic_beta"] = anthropic_beta_list
|
||||
|
||||
return _anthropic_request
|
||||
|
||||
def transform_response(
|
||||
|
|
|
|||
|
|
@ -190,13 +190,15 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
] = True # cohere requires stream = True in inference params
|
||||
request_data = {"prompt": prompt, **inference_params}
|
||||
elif provider == "anthropic":
|
||||
return litellm.AmazonAnthropicClaude3Config().transform_request(
|
||||
transformed_request = litellm.AmazonAnthropicClaudeConfig().transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
return transformed_request
|
||||
elif provider == "nova":
|
||||
return litellm.AmazonInvokeNovaConfig().transform_request(
|
||||
model=model,
|
||||
|
|
@ -293,7 +295,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
completion_response["generations"][0]["finish_reason"]
|
||||
)
|
||||
elif provider == "anthropic":
|
||||
return litellm.AmazonAnthropicClaude3Config().transform_response(
|
||||
return litellm.AmazonAnthropicClaudeConfig().transform_response(
|
||||
model=model,
|
||||
raw_response=raw_response,
|
||||
model_response=model_response,
|
||||
|
|
|
|||
|
|
@ -4,11 +4,14 @@ Common utilities used across bedrock chat/embedding/image generation
|
|||
|
||||
import json
|
||||
import os
|
||||
from typing import TYPE_CHECKING, List, Literal, Optional, Union
|
||||
from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Union
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
BaseAnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.secret_managers.main import get_secret
|
||||
|
|
@ -443,23 +446,82 @@ class BedrockModelInfo(BaseLLMModelInfo):
|
|||
"""
|
||||
Get the bedrock route for the given model.
|
||||
"""
|
||||
route_mappings: Dict[str, Literal["invoke", "converse_like", "converse", "agent"]] = {
|
||||
"invoke/": "invoke",
|
||||
"converse_like/": "converse_like",
|
||||
"converse/": "converse",
|
||||
"agent/": "agent"
|
||||
}
|
||||
|
||||
# Check explicit routes first
|
||||
for prefix, route_type in route_mappings.items():
|
||||
if prefix in model:
|
||||
return route_type
|
||||
|
||||
base_model = BedrockModelInfo.get_base_model(model)
|
||||
alt_model = BedrockModelInfo.get_non_litellm_routing_model_name(model=model)
|
||||
if "invoke/" in model:
|
||||
return "invoke"
|
||||
elif "converse_like" in model:
|
||||
return "converse_like"
|
||||
elif "converse/" in model:
|
||||
return "converse"
|
||||
elif "agent/" in model:
|
||||
return "agent"
|
||||
elif (
|
||||
if (
|
||||
base_model in litellm.bedrock_converse_models
|
||||
or alt_model in litellm.bedrock_converse_models
|
||||
):
|
||||
return "converse"
|
||||
return "invoke"
|
||||
|
||||
@staticmethod
|
||||
def _explicit_converse_route(model: str) -> bool:
|
||||
"""
|
||||
Check if the model is an explicit converse route.
|
||||
"""
|
||||
return "converse/" in model
|
||||
|
||||
@staticmethod
|
||||
def _explicit_invoke_route(model: str) -> bool:
|
||||
"""
|
||||
Check if the model is an explicit invoke route.
|
||||
"""
|
||||
return "invoke/" in model
|
||||
|
||||
@staticmethod
|
||||
def _explicit_agent_route(model: str) -> bool:
|
||||
"""
|
||||
Check if the model is an explicit agent route.
|
||||
"""
|
||||
return "agent/" in model
|
||||
|
||||
@staticmethod
|
||||
def _explicit_converse_like_route(model: str) -> bool:
|
||||
"""
|
||||
Check if the model is an explicit converse like route.
|
||||
"""
|
||||
return "converse_like/" in model
|
||||
|
||||
|
||||
@staticmethod
|
||||
def get_bedrock_provider_config_for_messages_api(model: str) -> Optional[BaseAnthropicMessagesConfig]:
|
||||
"""
|
||||
Get the bedrock provider config for the given model.
|
||||
|
||||
Only route to AmazonAnthropicClaude3MessagesConfig() for BaseMessagesConfig
|
||||
|
||||
All other routes should return None since they will go through litellm.completion
|
||||
"""
|
||||
|
||||
#########################################################
|
||||
# Converse routes should go through litellm.completion()
|
||||
if BedrockModelInfo._explicit_converse_route(model):
|
||||
return None
|
||||
|
||||
#########################################################
|
||||
# This goes through litellm.AmazonAnthropicClaude3MessagesConfig()
|
||||
# Since bedrock Invoke supports Native Anthropic Messages API
|
||||
#########################################################
|
||||
if "claude" in model:
|
||||
return litellm.AmazonAnthropicClaudeMessagesConfig()
|
||||
|
||||
#########################################################
|
||||
# These routes will go through litellm.completion()
|
||||
#########################################################
|
||||
return None
|
||||
|
||||
class BedrockEventStreamDecoderBase:
|
||||
"""
|
||||
|
|
@ -524,3 +586,25 @@ class BedrockEventStreamDecoderBase:
|
|||
return None
|
||||
|
||||
return chunk.decode() # type: ignore[no-any-return]
|
||||
|
||||
|
||||
def get_anthropic_beta_from_headers(headers: dict) -> List[str]:
|
||||
"""
|
||||
Extract anthropic-beta header values and convert them to a list.
|
||||
Supports comma-separated values from user headers.
|
||||
|
||||
Used by both converse and invoke transformations for consistent handling
|
||||
of anthropic-beta headers that should be passed to AWS Bedrock.
|
||||
|
||||
Args:
|
||||
headers (dict): Request headers dictionary
|
||||
|
||||
Returns:
|
||||
List[str]: List of anthropic beta feature strings, empty list if no header
|
||||
"""
|
||||
anthropic_beta_header = headers.get("anthropic-beta")
|
||||
if not anthropic_beta_header:
|
||||
return []
|
||||
|
||||
# Split comma-separated values and strip whitespace
|
||||
return [beta.strip() for beta in anthropic_beta_header.split(",")]
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder
|
|||
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
||||
AmazonInvokeConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.common_utils import get_anthropic_beta_from_headers
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import GenericStreamingChunk
|
||||
from litellm.types.utils import GenericStreamingChunk as GChunk
|
||||
|
|
@ -25,12 +26,13 @@ else:
|
|||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
class AmazonAnthropicClaude3MessagesConfig(
|
||||
class AmazonAnthropicClaudeMessagesConfig(
|
||||
AnthropicMessagesConfig,
|
||||
AmazonInvokeConfig,
|
||||
):
|
||||
"""
|
||||
Call Claude model family in the /v1/messages API spec
|
||||
Supports anthropic_beta parameter for beta features.
|
||||
"""
|
||||
|
||||
DEFAULT_BEDROCK_ANTHROPIC_API_VERSION = "bedrock-2023-05-31"
|
||||
|
|
@ -127,6 +129,12 @@ class AmazonAnthropicClaude3MessagesConfig(
|
|||
# 3. `model` is not allowed in request body for bedrock invoke
|
||||
if "model" in anthropic_messages_request:
|
||||
anthropic_messages_request.pop("model", None)
|
||||
|
||||
# 4. Handle anthropic_beta from user headers
|
||||
anthropic_beta_list = get_anthropic_beta_from_headers(headers)
|
||||
if anthropic_beta_list:
|
||||
anthropic_messages_request["anthropic_beta"] = anthropic_beta_list
|
||||
|
||||
return anthropic_messages_request
|
||||
|
||||
def get_async_streaming_response_iterator(
|
||||
|
|
|
|||
|
|
@ -21,6 +21,11 @@ from ...openai.chat.gpt_transformation import OpenAIGPTConfig
|
|||
|
||||
|
||||
class HostedVLLMChatConfig(OpenAIGPTConfig):
|
||||
def get_supported_openai_params(self, model: str) -> List[str]:
|
||||
params = super().get_supported_openai_params(model)
|
||||
params.append("reasoning_effort")
|
||||
return params
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ from litellm.types.utils import (
|
|||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
ProviderField,
|
||||
StreamingChoices,
|
||||
Delta,
|
||||
)
|
||||
|
||||
from ..common_utils import OllamaError, _convert_image
|
||||
|
|
@ -423,7 +425,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
|
|||
) -> Union[GenericStreamingChunk, ModelResponseStream]:
|
||||
return self.chunk_parser(json.loads(str_line))
|
||||
|
||||
def chunk_parser(self, chunk: dict) -> GenericStreamingChunk:
|
||||
def chunk_parser(self, chunk: dict) -> Union[GenericStreamingChunk, ModelResponseStream]:
|
||||
try:
|
||||
if "error" in chunk:
|
||||
raise Exception(f"Ollama Error - {chunk}")
|
||||
|
|
@ -459,6 +461,17 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
|
|||
finish_reason="stop",
|
||||
usage=None,
|
||||
)
|
||||
elif "thinking" in chunk and not chunk["response"]:
|
||||
# Return reasoning content as ModelResponseStream so UIs can render it
|
||||
thinking_content = chunk.get("thinking") or ""
|
||||
return ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(reasoning_content=thinking_content),
|
||||
)
|
||||
]
|
||||
)
|
||||
else:
|
||||
raise Exception(f"Unable to parse ollama chunk - {chunk}")
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
"""
|
||||
This file contains the calling Azure OpenAI's `/openai/realtime` endpoint.
|
||||
This file contains the calling OpenAI's `/v1/realtime` endpoint.
|
||||
|
||||
This requires websockets, and is currently only supported on LiteLLM Proxy.
|
||||
"""
|
||||
|
|
@ -15,7 +15,7 @@ from litellm.types.realtime import RealtimeQueryParams
|
|||
class OpenAIRealtime(OpenAIChatCompletion):
|
||||
def _construct_url(self, api_base: str, query_params: RealtimeQueryParams) -> str:
|
||||
"""
|
||||
Construct the backend websocket URL with all query parameters (excluding 'model' if present).
|
||||
Construct the backend websocket URL with all query parameters (including 'model').
|
||||
"""
|
||||
from httpx import URL
|
||||
|
||||
|
|
@ -24,10 +24,9 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
url = URL(api_base)
|
||||
# Set the correct path
|
||||
url = url.copy_with(path="/v1/realtime")
|
||||
# Build query dict excluding 'model'
|
||||
query_dict = {k: v for k, v in query_params.items() if k != "model"}
|
||||
if query_dict:
|
||||
url = url.copy_with(params=query_dict)
|
||||
# Include all query parameters including 'model'
|
||||
if query_params:
|
||||
url = url.copy_with(params=query_params)
|
||||
return str(url)
|
||||
|
||||
async def async_realtime(
|
||||
|
|
@ -43,11 +42,10 @@ class OpenAIRealtime(OpenAIChatCompletion):
|
|||
):
|
||||
import websockets
|
||||
from websockets.asyncio.client import ClientConnection
|
||||
|
||||
if api_base is None:
|
||||
raise ValueError("api_base is required for Azure OpenAI calls")
|
||||
api_base = "https://api.openai.com/"
|
||||
if api_key is None:
|
||||
raise ValueError("api_key is required for Azure OpenAI calls")
|
||||
raise ValueError("api_key is required for OpenAI realtime calls")
|
||||
|
||||
# Use all query params if provided, else fallback to just model
|
||||
if query_params is None:
|
||||
|
|
|
|||
|
|
@ -81,8 +81,18 @@ class VolcEngineConfig(OpenAILikeChatConfig):
|
|||
)
|
||||
|
||||
if "thinking" in optional_params:
|
||||
optional_params.setdefault("extra_body", {})["thinking"] = (
|
||||
optional_params.pop("thinking")
|
||||
)
|
||||
thinking_value = optional_params.pop("thinking")
|
||||
|
||||
# Handle disabled thinking case - don't add to extra_body if disabled
|
||||
if (
|
||||
thinking_value is not None
|
||||
and isinstance(thinking_value, dict)
|
||||
and thinking_value.get("type") == "disabled"
|
||||
):
|
||||
# Skip adding thinking parameter when it's disabled
|
||||
pass
|
||||
else:
|
||||
# Add thinking parameter to extra_body for all other cases
|
||||
optional_params.setdefault("extra_body", {})["thinking"] = thinking_value
|
||||
|
||||
return optional_params
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig
|
|||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.realtime_api.main import _realtime_health_check
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.secret_managers.main import get_secret_bool, get_secret_str
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import RawRequestTypedDict
|
||||
from litellm.utils import (
|
||||
|
|
@ -1592,18 +1592,10 @@ def completion( # type: ignore # noqa: PLR0915
|
|||
raise e
|
||||
|
||||
elif custom_llm_provider == "azure_ai":
|
||||
api_base = (
|
||||
api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
|
||||
or litellm.api_base
|
||||
or get_secret("AZURE_AI_API_BASE")
|
||||
)
|
||||
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
|
||||
api_base = AzureFoundryModelInfo.get_api_base(api_base)
|
||||
# set API KEY
|
||||
api_key = (
|
||||
api_key
|
||||
or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there
|
||||
or litellm.openai_key
|
||||
or get_secret("AZURE_AI_API_KEY")
|
||||
)
|
||||
api_key = AzureFoundryModelInfo.get_api_key(api_key)
|
||||
|
||||
headers = headers or litellm.headers
|
||||
|
||||
|
|
@ -1981,26 +1973,49 @@ def completion( # type: ignore # noqa: PLR0915
|
|||
optional_params[k] = v
|
||||
|
||||
## COMPLETION CALL
|
||||
use_base_llm_http_handler = get_secret_bool(
|
||||
"EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER"
|
||||
)
|
||||
try:
|
||||
response = openai_chat_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
timeout=timeout, # type: ignore
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
client=client, # pass AsyncOpenAI, OpenAI client
|
||||
organization=organization,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
if use_base_llm_http_handler:
|
||||
response = base_llm_http_handler.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
api_base=api_base,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_response=model_response,
|
||||
encoding=encoding,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
timeout=timeout,
|
||||
litellm_params=litellm_params,
|
||||
acompletion=acompletion,
|
||||
stream=stream,
|
||||
api_key=api_key,
|
||||
headers=headers,
|
||||
client=client,
|
||||
provider_config=provider_config,
|
||||
)
|
||||
else:
|
||||
response = openai_chat_completions.completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
headers=headers,
|
||||
model_response=model_response,
|
||||
print_verbose=print_verbose,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
timeout=timeout, # type: ignore
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
client=client, # pass AsyncOpenAI, OpenAI client
|
||||
organization=organization,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
except Exception as e:
|
||||
## LOGGING - log the original exception returned
|
||||
logging.post_call(
|
||||
|
|
@ -2981,7 +2996,7 @@ def completion( # type: ignore # noqa: PLR0915
|
|||
logger_fn=logger_fn,
|
||||
encoding=encoding,
|
||||
logging_obj=logging,
|
||||
extra_headers=extra_headers,
|
||||
extra_headers=headers, # Use merged headers instead of original extra_headers
|
||||
timeout=timeout,
|
||||
acompletion=acompletion,
|
||||
client=client,
|
||||
|
|
|
|||
|
|
@ -4790,6 +4790,24 @@
|
|||
],
|
||||
"source": "https://azuremarketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice"
|
||||
},
|
||||
"azure_ai/FLUX-1.1-pro": {
|
||||
"output_cost_per_image": 0.04,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "image_generation",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
],
|
||||
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659"
|
||||
},
|
||||
"azure_ai/FLUX.1-Kontext-pro": {
|
||||
"output_cost_per_image": 0.04,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "image_generation",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
],
|
||||
"source": "https://azuremarketplace.microsoft.com/pt-br/marketplace/apps/cohere.cohere-embed-4-offer?tab=PlansAndPrice"
|
||||
},
|
||||
"babbage-002": {
|
||||
"max_tokens": 16384,
|
||||
"max_input_tokens": 16384,
|
||||
|
|
@ -9686,6 +9704,21 @@
|
|||
"supports_assistant_prefill": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"vertex_ai/deepseek-ai/deepseek-r1-0528-maas": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 65336,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 1.35e-06,
|
||||
"output_cost_per_token": 5.4e-06,
|
||||
"litellm_provider": "vertex_ai-deepseek_models",
|
||||
"mode": "chat",
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"vertex_ai/meta/llama3-405b-instruct-maas": {
|
||||
"max_tokens": 32000,
|
||||
"max_input_tokens": 32000,
|
||||
|
|
@ -11190,6 +11223,17 @@
|
|||
"mode": "chat",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/deepseek/deepseek-chat-v3-0324": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 65536,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 1.4e-07,
|
||||
"output_cost_per_token": 2.8e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"supports_prompt_caching": true,
|
||||
"mode": "chat",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/deepseek/deepseek-coder": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 66000,
|
||||
|
|
|
|||
|
|
@ -236,12 +236,17 @@ class MCPRequestHandler:
|
|||
)
|
||||
|
||||
#########################################################
|
||||
# If team has mcp_servers, then key must have a subset of the team's mcp_servers
|
||||
# If team has mcp_servers, handle inheritance and intersection logic
|
||||
#########################################################
|
||||
if len(allowed_mcp_servers_for_team) > 0:
|
||||
for _mcp_server in allowed_mcp_servers_for_key:
|
||||
if _mcp_server in allowed_mcp_servers_for_team:
|
||||
allowed_mcp_servers.append(_mcp_server)
|
||||
if len(allowed_mcp_servers_for_key) > 0:
|
||||
# Key has its own MCP permissions - use intersection with team permissions
|
||||
for _mcp_server in allowed_mcp_servers_for_key:
|
||||
if _mcp_server in allowed_mcp_servers_for_team:
|
||||
allowed_mcp_servers.append(_mcp_server)
|
||||
else:
|
||||
# Key has no MCP permissions - inherit from team
|
||||
allowed_mcp_servers = allowed_mcp_servers_for_team
|
||||
else:
|
||||
allowed_mcp_servers = allowed_mcp_servers_for_key
|
||||
|
||||
|
|
|
|||
|
|
@ -295,6 +295,7 @@ class MCPServerManager:
|
|||
command=getattr(mcp_server, "command", None),
|
||||
args=getattr(mcp_server, "args", None) or [],
|
||||
env=env_dict,
|
||||
access_groups=getattr(mcp_server, "mcp_access_groups", None),
|
||||
)
|
||||
self.registry[mcp_server.server_id] = new_server
|
||||
verbose_logger.debug(f"Added MCP Server: {name_for_prefix}")
|
||||
|
|
@ -1050,7 +1051,9 @@ class MCPServerManager:
|
|||
auth_type=_server_config.auth_type,
|
||||
created_at=datetime.datetime.now(),
|
||||
updated_at=datetime.datetime.now(),
|
||||
description=_server_config.mcp_info.get("description") if _server_config.mcp_info else None,
|
||||
mcp_info=_server_config.mcp_info,
|
||||
mcp_access_groups=_server_config.access_groups or [],
|
||||
# Stdio-specific fields
|
||||
command=getattr(_server_config, "command", None),
|
||||
args=getattr(_server_config, "args", None) or [],
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1 @@
|
|||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{6580:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_b0dd8a', '__Inter_Fallback_b0dd8a'",fontStyle:"normal"},className:"__className_b0dd8a"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=6580)}),_N_E=n.O()}]);
|
||||
|
|
@ -1 +0,0 @@
|
|||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{96443:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_b0dd8a', '__Inter_Fallback_b0dd8a'",fontStyle:"normal"},className:"__className_b0dd8a"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=96443)}),_N_E=n.O()}]);
|
||||
|
|
@ -1 +1 @@
|
|||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{21024:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[85,487,154,162,971,117,744],function(){return e(e.s=21024)}),_N_E=e.O()}]);
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{11790:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[50,487,154,162,971,117,744],function(){return e(e.s=11790)}),_N_E=e.O()}]);
|
||||
|
|
@ -1 +1 @@
|
|||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{64563:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[85,487,866,154,162,172,971,117,744],function(){return e(e.s=64563)}),_N_E=e.O()}]);
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{58538:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,487,866,154,162,172,971,117,744],function(){return e(e.s=58538)}),_N_E=e.O()}]);
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +1 @@
|
|||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{10264:function(e,n,t){Promise.resolve().then(t.t.bind(t,12846,23)),Promise.resolve().then(t.t.bind(t,19107,23)),Promise.resolve().then(t.t.bind(t,61060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,36423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(54278),n(10264)}),_N_E=e.O()}]);
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{20169:function(e,n,t){Promise.resolve().then(t.t.bind(t,12846,23)),Promise.resolve().then(t.t.bind(t,19107,23)),Promise.resolve().then(t.t.bind(t,61060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,36423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(54278),n(20169)}),_N_E=e.O()}]);
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
2:I[19107,[],"ClientPageRoot"]
|
||||
3:I[39105,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","85","static/chunks/85-52b9060394399707.js","487","static/chunks/487-79ed94231812dae7.js","866","static/chunks/866-9e1803a09e9ae8da.js","247","static/chunks/247-7557228b7131016b.js","154","static/chunks/154-7bf3bbb913e68f71.js","162","static/chunks/162-f2925685093720a4.js","172","static/chunks/172-25e8f67ccf021150.js","931","static/chunks/app/page-c64580821d04b2c5.js"],"default",1]
|
||||
3:I[60750,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","50","static/chunks/50-d0da2dd7acce2eb9.js","487","static/chunks/487-79ed94231812dae7.js","866","static/chunks/866-3523e0e07cf314f6.js","682","static/chunks/682-52e4da3cfda27e50.js","154","static/chunks/154-686bf7931b450e3d.js","162","static/chunks/162-8e70dcd28bbbec09.js","172","static/chunks/172-c602ad2ba592907b.js","931","static/chunks/app/page-aeac1c2242be716a.js"],"default",1]
|
||||
4:I[4707,[],""]
|
||||
5:I[36423,[],""]
|
||||
0:["ILJ2l6ZzNB2f7RRsIZVJI",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/c8d591a6ccd18f71.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
0:["cx_klKzFUqmg6FLvftwd4",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/618290a32b9bb91a.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
2:I[19107,[],"ClientPageRoot"]
|
||||
3:I[52829,["85","static/chunks/85-52b9060394399707.js","487","static/chunks/487-79ed94231812dae7.js","154","static/chunks/154-7bf3bbb913e68f71.js","162","static/chunks/162-f2925685093720a4.js","418","static/chunks/app/model_hub/page-72c8a8dbd0d3984f.js"],"default",1]
|
||||
3:I[52829,["50","static/chunks/50-d0da2dd7acce2eb9.js","487","static/chunks/487-79ed94231812dae7.js","154","static/chunks/154-686bf7931b450e3d.js","162","static/chunks/162-8e70dcd28bbbec09.js","418","static/chunks/app/model_hub/page-6daa1df7fb2c7a9d.js"],"default",1]
|
||||
4:I[4707,[],""]
|
||||
5:I[36423,[],""]
|
||||
0:["ILJ2l6ZzNB2f7RRsIZVJI",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/c8d591a6ccd18f71.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
0:["cx_klKzFUqmg6FLvftwd4",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/618290a32b9bb91a.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
2:I[19107,[],"ClientPageRoot"]
|
||||
3:I[22775,["85","static/chunks/85-52b9060394399707.js","487","static/chunks/487-79ed94231812dae7.js","866","static/chunks/866-9e1803a09e9ae8da.js","154","static/chunks/154-7bf3bbb913e68f71.js","162","static/chunks/162-f2925685093720a4.js","172","static/chunks/172-25e8f67ccf021150.js","25","static/chunks/app/model_hub_table/page-10c235fc18df224f.js"],"default",1]
|
||||
3:I[22775,["50","static/chunks/50-d0da2dd7acce2eb9.js","487","static/chunks/487-79ed94231812dae7.js","866","static/chunks/866-3523e0e07cf314f6.js","154","static/chunks/154-686bf7931b450e3d.js","162","static/chunks/162-8e70dcd28bbbec09.js","172","static/chunks/172-c602ad2ba592907b.js","25","static/chunks/app/model_hub_table/page-aef41992dbd4614b.js"],"default",1]
|
||||
4:I[4707,[],""]
|
||||
5:I[36423,[],""]
|
||||
0:["ILJ2l6ZzNB2f7RRsIZVJI",[[["",{"children":["model_hub_table",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub_table",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub_table","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/c8d591a6ccd18f71.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
0:["cx_klKzFUqmg6FLvftwd4",[[["",{"children":["model_hub_table",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub_table",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub_table","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/618290a32b9bb91a.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
1
litellm/proxy/_experimental/out/onboarding.html
Normal file
1
litellm/proxy/_experimental/out/onboarding.html
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -1,7 +1,7 @@
|
|||
2:I[19107,[],"ClientPageRoot"]
|
||||
3:I[12011,["665","static/chunks/3014691f-b7b79b78e27792f3.js","85","static/chunks/85-52b9060394399707.js","154","static/chunks/154-7bf3bbb913e68f71.js","461","static/chunks/app/onboarding/page-6b5d568180da3ccd.js"],"default",1]
|
||||
3:I[12011,["665","static/chunks/3014691f-b7b79b78e27792f3.js","50","static/chunks/50-d0da2dd7acce2eb9.js","154","static/chunks/154-686bf7931b450e3d.js","461","static/chunks/app/onboarding/page-3c5840c907b0a5c8.js"],"default",1]
|
||||
4:I[4707,[],""]
|
||||
5:I[36423,[],""]
|
||||
0:["ILJ2l6ZzNB2f7RRsIZVJI",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/c8d591a6ccd18f71.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
0:["cx_klKzFUqmg6FLvftwd4",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/618290a32b9bb91a.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ model_list:
|
|||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
|
||||
litellm_settings:
|
||||
callbacks: ["otel"]
|
||||
cache: true
|
||||
cache_params:
|
||||
type: redis
|
||||
|
|
|
|||
|
|
@ -527,9 +527,20 @@ class LiteLLMRoutes(enum.Enum):
|
|||
"/organization/member_update",
|
||||
]
|
||||
|
||||
# Routes accessible by Admin Viewer (read-only admin access)
|
||||
admin_viewer_routes = [
|
||||
"/user/list",
|
||||
"/user/available_users",
|
||||
"/user/available_roles",
|
||||
"/user/daily/activity",
|
||||
"/team/daily/activity",
|
||||
"/tag/daily/activity",
|
||||
"/tag/list",
|
||||
] + info_routes
|
||||
|
||||
# All routes accesible by an Org Admin
|
||||
org_admin_allowed_routes = (
|
||||
org_admin_only_routes + management_routes + self_managed_routes
|
||||
org_admin_only_routes + management_routes + self_managed_routes + admin_viewer_routes
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1223,6 +1234,12 @@ class NewTeamRequest(TeamBase):
|
|||
team_member_budget: Optional[float] = (
|
||||
None # allow user to set a budget for all team members
|
||||
)
|
||||
team_member_rpm_limit: Optional[int] = (
|
||||
None # allow user to set RPM limit for all team members
|
||||
)
|
||||
team_member_tpm_limit: Optional[int] = (
|
||||
None # allow user to set TPM limit for all team members
|
||||
)
|
||||
team_member_key_duration: Optional[str] = None # e.g. "1d", "1w", "1m"
|
||||
|
||||
model_config = ConfigDict(protected_namespaces=())
|
||||
|
|
@ -1266,6 +1283,8 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase):
|
|||
guardrails: Optional[List[str]] = None
|
||||
object_permission: Optional[LiteLLM_ObjectPermissionBase] = None
|
||||
team_member_budget: Optional[float] = None
|
||||
team_member_rpm_limit: Optional[int] = None
|
||||
team_member_tpm_limit: Optional[int] = None
|
||||
team_member_key_duration: Optional[str] = None
|
||||
|
||||
|
||||
|
|
@ -1656,7 +1675,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
|
|||
)
|
||||
alert_to_webhook_url: Optional[Dict] = Field(
|
||||
None,
|
||||
description="Mapping of alert type to webhook url. e.g. `alert_to_webhook_url: {'budget_alerts': 'https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX'}`",
|
||||
description="Mapping of alert type to webhook url. e.g. `alert_to_webhook_url: {'budget_alerts': 'https://nothooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX'}`",
|
||||
)
|
||||
alerting_args: Optional[Dict] = Field(
|
||||
None, description="Controllable params for slack alerting - e.g. ttl in cache."
|
||||
|
|
@ -1758,10 +1777,14 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
|
|||
team_blocked: bool = False
|
||||
soft_budget: Optional[float] = None
|
||||
team_model_aliases: Optional[Dict] = None
|
||||
team_member_spend: Optional[float] = None
|
||||
team_member: Optional[Member] = None
|
||||
team_metadata: Optional[Dict] = None
|
||||
|
||||
# Team Member Specific Params
|
||||
team_member_spend: Optional[float] = None
|
||||
team_member_tpm_limit: Optional[int] = None
|
||||
team_member_rpm_limit: Optional[int] = None
|
||||
|
||||
# End User Params
|
||||
end_user_id: Optional[str] = None
|
||||
end_user_tpm_limit: Optional[int] = None
|
||||
|
|
@ -1850,8 +1873,7 @@ class UserAPIKeyAuth(
|
|||
key_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME,
|
||||
team_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME,
|
||||
)
|
||||
|
||||
|
||||
|
||||
class UserInfoResponse(LiteLLMPydanticObjectBase):
|
||||
user_id: Optional[str]
|
||||
user_info: Optional[Union[dict, BaseModel]]
|
||||
|
|
@ -2620,6 +2642,16 @@ class LiteLLM_TeamMembership(LiteLLMPydanticObjectBase):
|
|||
spend: Optional[float] = 0.0
|
||||
litellm_budget_table: Optional[LiteLLM_BudgetTable]
|
||||
|
||||
def safe_get_team_member_rpm_limit(self) -> Optional[int]:
|
||||
if self.litellm_budget_table is not None:
|
||||
return self.litellm_budget_table.rpm_limit
|
||||
return None
|
||||
|
||||
def safe_get_team_member_tpm_limit(self) -> Optional[int]:
|
||||
if self.litellm_budget_table is not None:
|
||||
return self.litellm_budget_table.tpm_limit
|
||||
return None
|
||||
|
||||
|
||||
#### Organization / Team Member Requests ####
|
||||
|
||||
|
|
@ -2730,11 +2762,22 @@ class TeamMemberDeleteRequest(MemberDeleteRequest):
|
|||
class TeamMemberUpdateRequest(TeamMemberDeleteRequest):
|
||||
max_budget_in_team: Optional[float] = None
|
||||
role: Optional[Literal["admin", "user"]] = None
|
||||
tpm_limit: Optional[int] = Field(
|
||||
default=None,
|
||||
description="Tokens per minute limit for this team member"
|
||||
)
|
||||
rpm_limit: Optional[int] = Field(
|
||||
default=None,
|
||||
description="Requests per minute limit for this team member"
|
||||
)
|
||||
|
||||
|
||||
|
||||
class TeamMemberUpdateResponse(MemberUpdateResponse):
|
||||
team_id: str
|
||||
max_budget_in_team: Optional[float] = None
|
||||
tpm_limit: Optional[int] = None
|
||||
rpm_limit: Optional[int] = None
|
||||
|
||||
|
||||
class TeamModelAddRequest(BaseModel):
|
||||
|
|
@ -2984,6 +3027,7 @@ class JWTAuthBuilderResult(TypedDict):
|
|||
user_id: Optional[str]
|
||||
end_user_id: Optional[str]
|
||||
org_id: Optional[str]
|
||||
team_membership: Optional[LiteLLM_TeamMembership]
|
||||
|
||||
|
||||
class ClientSideFallbackModel(TypedDict, total=False):
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from litellm.proxy._types import (
|
|||
LiteLLM_ObjectPermissionTable,
|
||||
LiteLLM_OrganizationMembershipTable,
|
||||
LiteLLM_OrganizationTable,
|
||||
LiteLLM_TeamMembership,
|
||||
LiteLLM_TeamTable,
|
||||
LiteLLM_TeamTableCachedObj,
|
||||
LiteLLM_UserTable,
|
||||
|
|
@ -501,6 +502,61 @@ async def get_end_user_object(
|
|||
return None
|
||||
|
||||
|
||||
@log_db_metrics
|
||||
async def get_team_membership(
|
||||
user_id: str,
|
||||
team_id: str,
|
||||
prisma_client: Optional[PrismaClient],
|
||||
user_api_key_cache: DualCache,
|
||||
parent_otel_span: Optional[Span] = None,
|
||||
proxy_logging_obj: Optional[ProxyLogging] = None,
|
||||
) -> Optional["LiteLLM_TeamMembership"]:
|
||||
"""
|
||||
Returns team membership object if user is member of team.
|
||||
|
||||
Do a isolated check for team membership vs. doing a combined key + team + user + team-membership check, as key might come in frequently for different users/teams. Larger call will slowdown query time. This way we get to cache the constant (key/team/user info) and only update based on the changing value (team membership).
|
||||
"""
|
||||
from litellm.proxy._types import LiteLLM_TeamMembership
|
||||
|
||||
if prisma_client is None:
|
||||
raise Exception("No db connected")
|
||||
|
||||
if user_id is None or team_id is None:
|
||||
return None
|
||||
|
||||
_key = "team_membership:{}:{}".format(user_id, team_id)
|
||||
|
||||
# check if in cache
|
||||
cached_membership_obj = await user_api_key_cache.async_get_cache(key=_key)
|
||||
if cached_membership_obj is not None:
|
||||
if isinstance(cached_membership_obj, dict):
|
||||
return LiteLLM_TeamMembership(**cached_membership_obj)
|
||||
elif isinstance(cached_membership_obj, LiteLLM_TeamMembership):
|
||||
return cached_membership_obj
|
||||
|
||||
# else, check db
|
||||
try:
|
||||
response = await prisma_client.db.litellm_teammembership.find_unique(
|
||||
where={"user_id_team_id": {"user_id": user_id, "team_id": team_id}},
|
||||
include={"litellm_budget_table": True},
|
||||
)
|
||||
|
||||
if response is None:
|
||||
return None
|
||||
|
||||
# save the team membership object to cache
|
||||
await user_api_key_cache.async_set_cache(
|
||||
key=_key, value=response
|
||||
)
|
||||
|
||||
_response = LiteLLM_TeamMembership(**response.dict())
|
||||
|
||||
return _response
|
||||
except Exception:
|
||||
verbose_proxy_logger.exception("Error getting team membership for user_id: %s, team_id: %s", user_id, team_id)
|
||||
return None
|
||||
|
||||
|
||||
def model_in_access_group(
|
||||
model: str, team_models: Optional[List[str]], llm_router: Optional[Router]
|
||||
) -> bool:
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from litellm.proxy._types import (
|
|||
LiteLLM_EndUserTable,
|
||||
LiteLLM_JWTAuth,
|
||||
LiteLLM_OrganizationTable,
|
||||
LiteLLM_TeamMembership,
|
||||
LiteLLM_TeamTable,
|
||||
LiteLLM_UserTable,
|
||||
LitellmUserRoles,
|
||||
|
|
@ -50,6 +51,7 @@ from .auth_checks import (
|
|||
get_org_object,
|
||||
get_role_based_models,
|
||||
get_role_based_routes,
|
||||
get_team_membership,
|
||||
get_team_object,
|
||||
get_user_object,
|
||||
)
|
||||
|
|
@ -707,6 +709,7 @@ class JWTAuthManager:
|
|||
user_id=user_id,
|
||||
end_user_id=None,
|
||||
org_id=org_id,
|
||||
team_membership=None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -839,6 +842,7 @@ class JWTAuthManager:
|
|||
user_email: Optional[str],
|
||||
org_id: Optional[str],
|
||||
end_user_id: Optional[str],
|
||||
team_id: Optional[str],
|
||||
valid_user_email: Optional[bool],
|
||||
jwt_handler: JWTHandler,
|
||||
prisma_client: Optional[PrismaClient],
|
||||
|
|
@ -850,6 +854,7 @@ class JWTAuthManager:
|
|||
Optional[LiteLLM_UserTable],
|
||||
Optional[LiteLLM_OrganizationTable],
|
||||
Optional[LiteLLM_EndUserTable],
|
||||
Optional[LiteLLM_TeamMembership],
|
||||
]:
|
||||
"""Get user, org, and end user objects"""
|
||||
org_object: Optional[LiteLLM_OrganizationTable] = None
|
||||
|
|
@ -899,8 +904,23 @@ class JWTAuthManager:
|
|||
if end_user_id
|
||||
else None
|
||||
)
|
||||
|
||||
team_membership_object: Optional[LiteLLM_TeamMembership] = None
|
||||
if user_id and team_id:
|
||||
team_membership_object = (
|
||||
await get_team_membership(
|
||||
user_id=user_id,
|
||||
team_id=team_id,
|
||||
prisma_client=prisma_client,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
parent_otel_span=parent_otel_span,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
if user_id and team_id
|
||||
else None
|
||||
)
|
||||
|
||||
return user_object, org_object, end_user_object
|
||||
return user_object, org_object, end_user_object, team_membership_object
|
||||
|
||||
@staticmethod
|
||||
def validate_object_id(
|
||||
|
|
@ -1125,11 +1145,12 @@ class JWTAuthManager:
|
|||
)
|
||||
|
||||
# Get other objects
|
||||
user_object, org_object, end_user_object = await JWTAuthManager.get_objects(
|
||||
user_object, org_object, end_user_object, team_membership_object = await JWTAuthManager.get_objects(
|
||||
user_id=user_id,
|
||||
user_email=user_email,
|
||||
org_id=org_id,
|
||||
end_user_id=end_user_id,
|
||||
team_id=team_id,
|
||||
valid_user_email=valid_user_email,
|
||||
jwt_handler=jwt_handler,
|
||||
prisma_client=prisma_client,
|
||||
|
|
@ -1165,6 +1186,8 @@ class JWTAuthManager:
|
|||
is_proxy_admin = True
|
||||
else:
|
||||
is_proxy_admin = False
|
||||
|
||||
|
||||
|
||||
return JWTAuthBuilderResult(
|
||||
is_proxy_admin=is_proxy_admin,
|
||||
|
|
@ -1177,4 +1200,5 @@ class JWTAuthManager:
|
|||
end_user_id=end_user_id,
|
||||
end_user_object=end_user_object,
|
||||
token=api_key,
|
||||
team_membership=team_membership_object,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -153,38 +153,11 @@ class RouteChecks:
|
|||
):
|
||||
pass
|
||||
elif _user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value:
|
||||
|
||||
if RouteChecks.is_llm_api_route(route=route):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"user not allowed to access this OpenAI routes, role= {_user_role}",
|
||||
)
|
||||
if RouteChecks.check_route_access(
|
||||
route=route, allowed_routes=LiteLLMRoutes.management_routes.value
|
||||
):
|
||||
|
||||
# the Admin Viewer is only allowed to call /user/update for their own user_id and can only update
|
||||
if route == "/user/update":
|
||||
# Check the Request params are valid for PROXY_ADMIN_VIEW_ONLY
|
||||
if request_data is not None and isinstance(request_data, dict):
|
||||
_params_updated = request_data.keys()
|
||||
for param in _params_updated:
|
||||
if param not in ["user_email", "password"]:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email and password can be updated",
|
||||
)
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route}",
|
||||
)
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route}",
|
||||
)
|
||||
|
||||
RouteChecks._check_proxy_admin_viewer_access(
|
||||
route=route,
|
||||
_user_role=_user_role,
|
||||
request_data=request_data,
|
||||
)
|
||||
elif (
|
||||
_user_role == LitellmUserRoles.INTERNAL_USER.value
|
||||
and RouteChecks.check_route_access(
|
||||
|
|
@ -401,3 +374,53 @@ class RouteChecks:
|
|||
if "streamGenerateContent" in route:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _check_proxy_admin_viewer_access(
|
||||
route: str,
|
||||
_user_role: str,
|
||||
request_data: dict,
|
||||
) -> None:
|
||||
"""
|
||||
Check access for PROXY_ADMIN_VIEW_ONLY role
|
||||
"""
|
||||
if RouteChecks.is_llm_api_route(route=route):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"user not allowed to access this OpenAI routes, role= {_user_role}",
|
||||
)
|
||||
|
||||
# Check if this is a write operation on management routes
|
||||
if RouteChecks.check_route_access(
|
||||
route=route, allowed_routes=LiteLLMRoutes.management_routes.value
|
||||
):
|
||||
# For management routes, only allow read operations or specific allowed updates
|
||||
if route == "/user/update":
|
||||
# Check the Request params are valid for PROXY_ADMIN_VIEW_ONLY
|
||||
if request_data is not None and isinstance(request_data, dict):
|
||||
_params_updated = request_data.keys()
|
||||
for param in _params_updated:
|
||||
if param not in ["user_email", "password"]:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route} and updating invalid param: {param}. only user_email and password can be updated",
|
||||
)
|
||||
elif route in ["/user/new", "/user/delete", "/team/new", "/team/update", "/team/delete", "/model/new", "/model/update", "/model/delete", "/key/generate", "/key/delete", "/key/update", "/key/regenerate", "/key/service-account/generate", "/key/block", "/key/unblock"] or route.startswith("/key/") and route.endswith("/regenerate"):
|
||||
# Block write operations for PROXY_ADMIN_VIEW_ONLY
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route}",
|
||||
)
|
||||
# Allow read operations on management routes (like /user/info, /team/info, /model/info)
|
||||
return
|
||||
elif RouteChecks.check_route_access(
|
||||
route=route, allowed_routes=LiteLLMRoutes.admin_viewer_routes.value
|
||||
):
|
||||
# Allow access to admin viewer routes (read-only admin endpoints)
|
||||
return
|
||||
else:
|
||||
# For other routes, block access
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"user not allowed to access this route, role= {_user_role}. Trying to access: {route}",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -502,6 +502,7 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
|
|||
end_user_object = result["end_user_object"]
|
||||
org_id = result["org_id"]
|
||||
token = result["token"]
|
||||
team_membership: Optional[LiteLLM_TeamMembership] = result.get("team_membership", None)
|
||||
|
||||
global_proxy_spend = await get_global_proxy_spend(
|
||||
litellm_proxy_admin_name=litellm_proxy_admin_name,
|
||||
|
|
@ -536,6 +537,10 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
|
|||
org_id=org_id,
|
||||
parent_otel_span=parent_otel_span,
|
||||
end_user_id=end_user_id,
|
||||
user_tpm_limit=user_object.tpm_limit if user_object is not None else None,
|
||||
user_rpm_limit=user_object.rpm_limit if user_object is not None else None,
|
||||
team_member_rpm_limit=team_membership.safe_get_team_member_rpm_limit() if team_membership is not None else None,
|
||||
team_member_tpm_limit=team_membership.safe_get_team_member_tpm_limit() if team_membership is not None else None,
|
||||
)
|
||||
# run through common checks
|
||||
_ = await common_checks(
|
||||
|
|
|
|||
|
|
@ -236,7 +236,7 @@ class KeysManagementClient:
|
|||
UnauthorizedError: If the request fails with a 401 status code
|
||||
requests.exceptions.RequestException: If the request fails with any other error
|
||||
"""
|
||||
url = f"{self._base_url}/keys/info?key={key}"
|
||||
url = f"{self._base_url}/key/info?key={key}"
|
||||
request = requests.Request("GET", url, headers=self._get_headers())
|
||||
|
||||
if return_request:
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ def remove_sensitive_info_from_deployment(deployment_dict: dict) -> dict:
|
|||
dict: The modified deployment dictionary with sensitive information removed.
|
||||
"""
|
||||
deployment_dict["litellm_params"].pop("api_key", None)
|
||||
deployment_dict["litellm_params"].pop("client_secret", None)
|
||||
deployment_dict["litellm_params"].pop("vertex_credentials", None)
|
||||
deployment_dict["litellm_params"].pop("aws_access_key_id", None)
|
||||
deployment_dict["litellm_params"].pop("aws_secret_access_key", None)
|
||||
|
|
|
|||
|
|
@ -448,6 +448,21 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
},
|
||||
)
|
||||
)
|
||||
|
||||
# Team Member rate limits
|
||||
if user_api_key_dict.user_id and (user_api_key_dict.team_member_rpm_limit is not None or user_api_key_dict.team_member_tpm_limit is not None):
|
||||
team_member_value = f"{user_api_key_dict.team_id}:{user_api_key_dict.user_id}"
|
||||
descriptors.append(
|
||||
RateLimitDescriptor(
|
||||
key="team_member",
|
||||
value=team_member_value,
|
||||
rate_limit={
|
||||
"requests_per_unit": user_api_key_dict.team_member_rpm_limit,
|
||||
"tokens_per_unit": user_api_key_dict.team_member_tpm_limit,
|
||||
"window_size": self.window_size,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
# End user rate limits
|
||||
if user_api_key_dict.end_user_id and (
|
||||
|
|
@ -662,6 +677,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
total_tokens=total_tokens,
|
||||
)
|
||||
)
|
||||
# Team Member TPM
|
||||
if user_api_key_team_id and user_api_key_user_id:
|
||||
pipeline_operations.extend(
|
||||
self._create_pipeline_operations(
|
||||
key="team_member",
|
||||
value=f"{user_api_key_team_id}:{user_api_key_user_id}",
|
||||
rate_limit_type="tokens",
|
||||
total_tokens=total_tokens,
|
||||
)
|
||||
)
|
||||
|
||||
# End User TPM
|
||||
if user_api_key_end_user_id:
|
||||
|
|
|
|||
|
|
@ -1,357 +0,0 @@
|
|||
[92m18:10:09 - LiteLLM Router:INFO[0m: router.py:660 - Routing strategy: simple-shuffle
|
||||
[92m18:10:11 - LiteLLM Proxy:INFO[0m: utils.py:1317 - All necessary views exist!
|
||||
[92m18:10:11 - LiteLLM Router:WARNING[0m: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments.
|
||||
[92m18:10:11 - LiteLLM Router:WARNING[0m: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments.
|
||||
[92m18:10:23 - LiteLLM Proxy:INFO[0m: ui_sso.py:129 - Redirecting to SSO login for http://localhost:4000/sso/callback
|
||||
[92m18:10:27 - LiteLLM Proxy:INFO[0m: ui_sso.py:495 - Starting SSO callback
|
||||
[92m18:10:27 - LiteLLM Proxy:INFO[0m: ui_sso.py:550 - Redirecting to http://localhost:4000/sso/callback
|
||||
[92m18:10:28 - LiteLLM Proxy:INFO[0m: ui_sso.py:581 - SSO callback result: id='krrishd' email='krrishdholakia@gmail.com' first_name=None last_name=None display_name='a3f1c107-04dc-4c93-ae60-7f32eb4b05ce' picture=None provider=None team_ids=[]
|
||||
[92m18:10:28 - LiteLLM Proxy:INFO[0m: ui_sso.py:671 - user_defined_values for creating ui key: {'models': [], 'user_id': 'krrishd', 'user_email': 'krrishdholakia@gmail.com', 'max_budget': None, 'user_role': 'proxy_admin', 'budget_duration': None}
|
||||
[92m18:10:28 - LiteLLM Proxy:INFO[0m: utils.py:1856 - Data Inserted into Keys Table
|
||||
[92m18:10:28 - LiteLLM Proxy:INFO[0m: ui_sso.py:761 - user_id: krrishd; jwt_token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoia3JyaXNoZCIsImtleSI6InNrLTVvOXVVc0ZaaTVBRFBiWERoanhCZlEiLCJ1c2VyX2VtYWlsIjoia3JyaXNoZGhvbGFraWFAZ21haWwuY29tIiwidXNlcl9yb2xlIjoicHJveHlfYWRtaW4iLCJsb2dpbl9tZXRob2QiOiJzc28iLCJwcmVtaXVtX3VzZXIiOnRydWUsImF1dGhfaGVhZGVyX25hbWUiOiJBdXRob3JpemF0aW9uIiwiZGlzYWJsZWRfbm9uX2FkbWluX3BlcnNvbmFsX2tleV9jcmVhdGlvbiI6ZmFsc2UsInNlcnZlcl9yb290X3BhdGgiOiIvIn0.OiZdFjZ2wiMhFbMCwu2cZYXh7oV5BB8Vta-Ysk5JBQU
|
||||
[92m18:10:28 - LiteLLM Proxy:INFO[0m: ui_sso.py:764 - Redirecting to http://localhost:4000/ui/?login=success
|
||||
[92m18:10:30 - LiteLLM Proxy:ERROR[0m: key_management_endpoints.py:2275 - Error in list_keys: Server disconnected without sending a response.
|
||||
Traceback (most recent call last):
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions
|
||||
yield
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 394, in handle_async_request
|
||||
resp = await self._pool.handle_async_request(req)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 216, in handle_async_request
|
||||
raise exc from None
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 196, in handle_async_request
|
||||
response = await connection.handle_async_request(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 101, in handle_async_request
|
||||
return await self._connection.handle_async_request(request)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 143, in handle_async_request
|
||||
raise exc
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 113, in handle_async_request
|
||||
) = await self._receive_response_headers(**kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 186, in _receive_response_headers
|
||||
event = await self._receive_event(timeout=timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 238, in _receive_event
|
||||
raise RemoteProtocolError(msg)
|
||||
httpcore.RemoteProtocolError: Server disconnected without sending a response.
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/management_endpoints/key_management_endpoints.py", line 2255, in list_keys
|
||||
response = await _list_key_helper(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/management_endpoints/key_management_endpoints.py", line 2434, in _list_key_helper
|
||||
total_count = await prisma_client.db.litellm_verificationtoken.count(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/actions.py", line 10157, in count
|
||||
resp = await self._client._execute(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_base_client.py", line 543, in _execute
|
||||
return await self._engine.query(builder.build(), tx_id=self._tx_id)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_query.py", line 402, in query
|
||||
return await self.request(
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_http.py", line 217, in request
|
||||
response = await self.session.request(method, url, **kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_async_http.py", line 26, in request
|
||||
return Response(await self.session.request(method, url, **kwargs))
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1540, in request
|
||||
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1629, in send
|
||||
response = await self._send_handling_auth(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1657, in _send_handling_auth
|
||||
response = await self._send_handling_redirects(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects
|
||||
response = await self._send_single_request(request)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1730, in _send_single_request
|
||||
response = await transport.handle_async_request(request)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 393, in handle_async_request
|
||||
with map_httpcore_exceptions():
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__
|
||||
self.gen.throw(typ, value, traceback)
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
|
||||
raise mapped_exc(message) from exc
|
||||
httpx.RemoteProtocolError: Server disconnected without sending a response.
|
||||
[92m18:10:30 - LiteLLM Proxy:ERROR[0m: proxy_server.py:2730 - litellm.proxy_server.py::add_deployment() - Error getting new models from DB - All connection attempts failed
|
||||
Traceback (most recent call last):
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions
|
||||
yield
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 394, in handle_async_request
|
||||
resp = await self._pool.handle_async_request(req)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 216, in handle_async_request
|
||||
raise exc from None
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 196, in handle_async_request
|
||||
response = await connection.handle_async_request(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 99, in handle_async_request
|
||||
raise exc
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 76, in handle_async_request
|
||||
stream = await self._connect(request)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 122, in _connect
|
||||
stream = await self._network_backend.connect_tcp(**kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_backends/auto.py", line 30, in connect_tcp
|
||||
return await self._backend.connect_tcp(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_backends/anyio.py", line 112, in connect_tcp
|
||||
with map_exceptions(exc_map):
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__
|
||||
self.gen.throw(typ, value, traceback)
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions
|
||||
raise to_exc(exc) from exc
|
||||
httpcore.ConnectError: All connection attempts failed
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/proxy_server.py", line 2728, in _get_models_from_db
|
||||
new_models = await prisma_client.db.litellm_proxymodeltable.find_many()
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/actions.py", line 2540, in find_many
|
||||
resp = await self._client._execute(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_base_client.py", line 543, in _execute
|
||||
return await self._engine.query(builder.build(), tx_id=self._tx_id)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_query.py", line 402, in query
|
||||
return await self.request(
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_http.py", line 217, in request
|
||||
response = await self.session.request(method, url, **kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_async_http.py", line 26, in request
|
||||
return Response(await self.session.request(method, url, **kwargs))
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1540, in request
|
||||
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1629, in send
|
||||
response = await self._send_handling_auth(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1657, in _send_handling_auth
|
||||
response = await self._send_handling_redirects(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects
|
||||
response = await self._send_single_request(request)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1730, in _send_single_request
|
||||
response = await transport.handle_async_request(request)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 393, in handle_async_request
|
||||
with map_httpcore_exceptions():
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__
|
||||
self.gen.throw(typ, value, traceback)
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
|
||||
raise mapped_exc(message) from exc
|
||||
httpx.ConnectError: All connection attempts failed
|
||||
[92m18:10:30 - LiteLLM Proxy:ERROR[0m: utils.py:1404 - LiteLLM Prisma Client Exception get_generic_data: All connection attempts failed
|
||||
[92m18:10:30 - LiteLLM Proxy:ERROR[0m: utils.py:1404 - LiteLLM Prisma Client Exception get_generic_data: All connection attempts failed
|
||||
[92m18:10:30 - LiteLLM Proxy:ERROR[0m: proxy_server.py:2778 - litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - All connection attempts failed
|
||||
Traceback (most recent call last):
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions
|
||||
yield
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 394, in handle_async_request
|
||||
resp = await self._pool.handle_async_request(req)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 216, in handle_async_request
|
||||
raise exc from None
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 196, in handle_async_request
|
||||
response = await connection.handle_async_request(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 99, in handle_async_request
|
||||
raise exc
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 76, in handle_async_request
|
||||
stream = await self._connect(request)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 122, in _connect
|
||||
stream = await self._network_backend.connect_tcp(**kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_backends/auto.py", line 30, in connect_tcp
|
||||
return await self._backend.connect_tcp(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_backends/anyio.py", line 112, in connect_tcp
|
||||
with map_exceptions(exc_map):
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__
|
||||
self.gen.throw(typ, value, traceback)
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions
|
||||
raise to_exc(exc) from exc
|
||||
httpcore.ConnectError: All connection attempts failed
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/proxy_server.py", line 2760, in add_deployment
|
||||
await self._update_llm_router(
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/proxy_server.py", line 2418, in _update_llm_router
|
||||
config_data = await proxy_config.get_config()
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/proxy_server.py", line 1584, in get_config
|
||||
config = await self._update_config_from_db(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/proxy_server.py", line 2706, in _update_config_from_db
|
||||
responses = await asyncio.gather(*_tasks)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/db/log_db_metrics.py", line 99, in wrapper
|
||||
raise e
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/db/log_db_metrics.py", line 42, in wrapper
|
||||
result = await func(*args, **kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/backoff/_async.py", line 151, in retry
|
||||
ret = await target(*args, **kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/utils.py", line 1418, in get_generic_data
|
||||
raise e
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/utils.py", line 1392, in get_generic_data
|
||||
response = await self.db.litellm_config.find_first( # type: ignore
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/actions.py", line 11822, in find_first
|
||||
resp = await self._client._execute(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_base_client.py", line 543, in _execute
|
||||
return await self._engine.query(builder.build(), tx_id=self._tx_id)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_query.py", line 402, in query
|
||||
return await self.request(
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_http.py", line 217, in request
|
||||
response = await self.session.request(method, url, **kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_async_http.py", line 26, in request
|
||||
return Response(await self.session.request(method, url, **kwargs))
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1540, in request
|
||||
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1629, in send
|
||||
response = await self._send_handling_auth(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1657, in _send_handling_auth
|
||||
response = await self._send_handling_redirects(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects
|
||||
response = await self._send_single_request(request)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1730, in _send_single_request
|
||||
response = await transport.handle_async_request(request)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 393, in handle_async_request
|
||||
with map_httpcore_exceptions():
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__
|
||||
self.gen.throw(typ, value, traceback)
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
|
||||
raise mapped_exc(message) from exc
|
||||
httpx.ConnectError: All connection attempts failed
|
||||
[92m18:10:30 - LiteLLM Proxy:ERROR[0m: utils.py:1404 - LiteLLM Prisma Client Exception get_generic_data: All connection attempts failed
|
||||
[92m18:10:30 - LiteLLM Proxy:ERROR[0m: utils.py:1404 - LiteLLM Prisma Client Exception get_generic_data: All connection attempts failed
|
||||
[92m18:10:30 - LiteLLM Proxy:INFO[0m: proxy_server.py:490 - Shutting down LiteLLM Proxy Server
|
||||
[92m18:11:47 - LiteLLM Router:INFO[0m: router.py:660 - Routing strategy: simple-shuffle
|
||||
[92m18:11:49 - LiteLLM Proxy:INFO[0m: utils.py:1317 - All necessary views exist!
|
||||
[92m18:11:50 - LiteLLM Router:WARNING[0m: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments.
|
||||
[92m18:11:50 - LiteLLM Router:WARNING[0m: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments.
|
||||
[92m18:12:00 - LiteLLM Proxy:ERROR[0m: proxy_server.py:2925 - litellm.proxy_server.py::get_credentials() - Error getting credentials from DB - Server disconnected without sending a response.
|
||||
Traceback (most recent call last):
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 101, in map_httpcore_exceptions
|
||||
yield
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 394, in handle_async_request
|
||||
resp = await self._pool.handle_async_request(req)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 216, in handle_async_request
|
||||
raise exc from None
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection_pool.py", line 196, in handle_async_request
|
||||
response = await connection.handle_async_request(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/connection.py", line 101, in handle_async_request
|
||||
return await self._connection.handle_async_request(request)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 143, in handle_async_request
|
||||
raise exc
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 113, in handle_async_request
|
||||
) = await self._receive_response_headers(**kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 186, in _receive_response_headers
|
||||
event = await self._receive_event(timeout=timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpcore/_async/http11.py", line 238, in _receive_event
|
||||
raise RemoteProtocolError(msg)
|
||||
httpcore.RemoteProtocolError: Server disconnected without sending a response.
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/proxy/proxy_server.py", line 2916, in get_credentials
|
||||
credentials = await prisma_client.db.litellm_credentialstable.find_many()
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/actions.py", line 1502, in find_many
|
||||
resp = await self._client._execute(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_base_client.py", line 543, in _execute
|
||||
return await self._engine.query(builder.build(), tx_id=self._tx_id)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_query.py", line 402, in query
|
||||
return await self.request(
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/engine/_http.py", line 217, in request
|
||||
response = await self.session.request(method, url, **kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/prisma/_async_http.py", line 26, in request
|
||||
return Response(await self.session.request(method, url, **kwargs))
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1540, in request
|
||||
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1629, in send
|
||||
response = await self._send_handling_auth(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1657, in _send_handling_auth
|
||||
response = await self._send_handling_redirects(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1694, in _send_handling_redirects
|
||||
response = await self._send_single_request(request)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_client.py", line 1730, in _send_single_request
|
||||
response = await transport.handle_async_request(request)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 393, in handle_async_request
|
||||
with map_httpcore_exceptions():
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/contextlib.py", line 155, in __exit__
|
||||
self.gen.throw(typ, value, traceback)
|
||||
File "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/httpx/_transports/default.py", line 118, in map_httpcore_exceptions
|
||||
raise mapped_exc(message) from exc
|
||||
httpx.RemoteProtocolError: Server disconnected without sending a response.
|
||||
[92m18:12:01 - LiteLLM Proxy:INFO[0m: proxy_server.py:490 - Shutting down LiteLLM Proxy Server
|
||||
[92m18:12:14 - LiteLLM Router:INFO[0m: router.py:660 - Routing strategy: simple-shuffle
|
||||
[92m18:12:16 - LiteLLM Proxy:INFO[0m: utils.py:1317 - All necessary views exist!
|
||||
[92m18:12:16 - LiteLLM Router:WARNING[0m: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments.
|
||||
[92m18:12:16 - LiteLLM Router:WARNING[0m: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments.
|
||||
[92m18:12:21 - LiteLLM Proxy:INFO[0m: ui_sso.py:129 - Redirecting to SSO login for http://localhost:4000/sso/callback
|
||||
[92m18:12:26 - LiteLLM Proxy:INFO[0m: ui_sso.py:495 - Starting SSO callback
|
||||
[92m18:12:26 - LiteLLM Proxy:INFO[0m: ui_sso.py:550 - Redirecting to http://localhost:4000/sso/callback
|
||||
[92m18:12:26 - LiteLLM Proxy:INFO[0m: ui_sso.py:581 - SSO callback result: id='krrishd' email='krrishdholakia@gmail.com' first_name=None last_name=None display_name='a3f1c107-04dc-4c93-ae60-7f32eb4b05ce' picture=None provider=None team_ids=[]
|
||||
[92m18:12:27 - LiteLLM Proxy:INFO[0m: ui_sso.py:672 - user_defined_values for creating ui key: {'models': [], 'user_id': 'krrishd', 'user_email': 'krrishdholakia@gmail.com', 'max_budget': None, 'user_role': 'proxy_admin', 'budget_duration': None}
|
||||
[92m18:12:27 - LiteLLM Proxy:INFO[0m: utils.py:1856 - Data Inserted into Keys Table
|
||||
[92m18:12:27 - LiteLLM Proxy:INFO[0m: ui_sso.py:762 - user_id: krrishd; jwt_token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoia3JyaXNoZCIsImtleSI6InNrLUQzMEFpdW9lckU3YlMyakFXWVFLd1EiLCJ1c2VyX2VtYWlsIjoia3JyaXNoZGhvbGFraWFAZ21haWwuY29tIiwidXNlcl9yb2xlIjoicHJveHlfYWRtaW4iLCJsb2dpbl9tZXRob2QiOiJzc28iLCJwcmVtaXVtX3VzZXIiOnRydWUsImF1dGhfaGVhZGVyX25hbWUiOiJBdXRob3JpemF0aW9uIiwiZGlzYWJsZWRfbm9uX2FkbWluX3BlcnNvbmFsX2tleV9jcmVhdGlvbiI6ZmFsc2UsInNlcnZlcl9yb290X3BhdGgiOiIvIn0.EzYP86hw12J4WHLe6ZZz4YgVNGPnxM_PHqLjINH2_-U
|
||||
[92m18:12:27 - LiteLLM Proxy:INFO[0m: ui_sso.py:765 - Redirecting to http://localhost:4000/ui/?login=success
|
||||
[92m18:12:31 - LiteLLM Proxy:INFO[0m: proxy_server.py:490 - Shutting down LiteLLM Proxy Server
|
||||
[92m18:15:07 - LiteLLM Router:INFO[0m: router.py:660 - Routing strategy: simple-shuffle
|
||||
[92m18:15:09 - LiteLLM Proxy:INFO[0m: utils.py:1317 - All necessary views exist!
|
||||
[92m18:15:09 - LiteLLM Router:WARNING[0m: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments.
|
||||
[92m18:15:09 - LiteLLM Router:WARNING[0m: router.py:4862 - Error upserting deployment: vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints., ignoring and continuing with other deployments.
|
||||
[92m18:15:17 - LiteLLM Proxy:INFO[0m: utils.py:1916 - Data Inserted into Config Table
|
||||
[92m18:15:28 - LiteLLM Proxy:INFO[0m: ui_sso.py:129 - Redirecting to SSO login for http://localhost:4000/sso/callback
|
||||
[92m18:15:32 - LiteLLM Proxy:INFO[0m: ui_sso.py:495 - Starting SSO callback
|
||||
[92m18:15:32 - LiteLLM Proxy:INFO[0m: ui_sso.py:550 - Redirecting to http://localhost:4000/sso/callback
|
||||
[92m18:15:32 - LiteLLM Proxy:INFO[0m: ui_sso.py:581 - SSO callback result: id='krrishd' email='krrishdholakia@gmail.com' first_name=None last_name=None display_name='a3f1c107-04dc-4c93-ae60-7f32eb4b05ce' picture=None provider=None team_ids=[]
|
||||
[92m18:15:37 - LiteLLM Proxy:INFO[0m: proxy_server.py:490 - Shutting down LiteLLM Proxy Server
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
from typing import Any, Optional, Union
|
||||
from typing import Any, Dict, Optional, Union
|
||||
|
||||
from litellm.proxy._types import (
|
||||
KeyRequestBase,
|
||||
|
|
@ -56,6 +56,8 @@ async def _upsert_budget_and_membership(
|
|||
max_budget: Optional[float],
|
||||
existing_budget_id: Optional[str],
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
tpm_limit: Optional[int] = None,
|
||||
rpm_limit: Optional[int] = None,
|
||||
):
|
||||
"""
|
||||
Helper function to Create/Update or Delete the budget within the team membership
|
||||
|
|
@ -66,33 +68,34 @@ async def _upsert_budget_and_membership(
|
|||
max_budget: The maximum budget for the team
|
||||
existing_budget_id: The ID of the existing budget, if any
|
||||
user_api_key_dict: User API Key dictionary containing user information
|
||||
tpm_limit: Tokens per minute limit for the team member
|
||||
rpm_limit: Requests per minute limit for the team member
|
||||
|
||||
If max_budget is None, the user's budget is removed from the team membership.
|
||||
If max_budget exists, a budget is updated or created and linked to the team membership.
|
||||
If max_budget, tpm_limit, and rpm_limit are all None, the user's budget is removed from the team membership.
|
||||
If any of these values exist, a budget is updated or created and linked to the team membership.
|
||||
"""
|
||||
if max_budget is None:
|
||||
# disconnect the budget since max_budget is None
|
||||
if max_budget is None and tpm_limit is None and rpm_limit is None:
|
||||
# disconnect the budget since all limits are None
|
||||
await tx.litellm_teammembership.update(
|
||||
where={"user_id_team_id": {"user_id": user_id, "team_id": team_id}},
|
||||
data={"litellm_budget_table": {"disconnect": True}},
|
||||
)
|
||||
return
|
||||
|
||||
if existing_budget_id:
|
||||
# update the existing budget
|
||||
await tx.litellm_budgettable.update(
|
||||
where={"budget_id": existing_budget_id},
|
||||
data={"max_budget": max_budget},
|
||||
)
|
||||
return
|
||||
|
||||
# create a new budget
|
||||
create_data: Dict[str, Any] = {
|
||||
"created_by": user_api_key_dict.user_id or "",
|
||||
"updated_by": user_api_key_dict.user_id or "",
|
||||
}
|
||||
if max_budget is not None:
|
||||
create_data["max_budget"] = max_budget
|
||||
if tpm_limit is not None:
|
||||
create_data["tpm_limit"] = tpm_limit
|
||||
if rpm_limit is not None:
|
||||
create_data["rpm_limit"] = rpm_limit
|
||||
|
||||
new_budget = await tx.litellm_budgettable.create(
|
||||
data={
|
||||
"max_budget": max_budget,
|
||||
"created_by": user_api_key_dict.user_id or "",
|
||||
"updated_by": user_api_key_dict.user_id or "",
|
||||
},
|
||||
data=create_data,
|
||||
include={"team_membership": True},
|
||||
)
|
||||
# upsert the team membership with the new/updated budget
|
||||
|
|
|
|||
|
|
@ -103,6 +103,137 @@ from litellm.types.proxy.management_endpoints.team_endpoints import (
|
|||
router = APIRouter()
|
||||
|
||||
|
||||
class TeamMemberBudgetHandler:
|
||||
"""Helper class to handle team member budget, RPM, and TPM limit operations"""
|
||||
|
||||
@staticmethod
|
||||
def should_create_budget(
|
||||
team_member_budget: Optional[float] = None,
|
||||
team_member_rpm_limit: Optional[int] = None,
|
||||
team_member_tpm_limit: Optional[int] = None,
|
||||
) -> bool:
|
||||
"""Check if any team member limits are provided"""
|
||||
return any([
|
||||
team_member_budget is not None,
|
||||
team_member_rpm_limit is not None,
|
||||
team_member_tpm_limit is not None,
|
||||
])
|
||||
|
||||
@staticmethod
|
||||
async def create_team_member_budget_table(
|
||||
data: Union[NewTeamRequest, LiteLLM_TeamTable],
|
||||
new_team_data_json: dict,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
team_member_budget: Optional[float] = None,
|
||||
team_member_rpm_limit: Optional[int] = None,
|
||||
team_member_tpm_limit: Optional[int] = None,
|
||||
) -> dict:
|
||||
"""Create team member budget table with provided limits"""
|
||||
from litellm.proxy._types import BudgetNewRequest
|
||||
from litellm.proxy.management_endpoints.budget_management_endpoints import (
|
||||
new_budget,
|
||||
)
|
||||
|
||||
if data.team_alias is not None:
|
||||
budget_id = (
|
||||
f"team-{data.team_alias.replace(' ', '-')}-budget-{uuid.uuid4().hex}"
|
||||
)
|
||||
else:
|
||||
budget_id = f"team-budget-{uuid.uuid4().hex}"
|
||||
|
||||
# Create budget request with all provided limits
|
||||
budget_request = BudgetNewRequest(
|
||||
budget_id=budget_id,
|
||||
budget_duration=data.budget_duration,
|
||||
)
|
||||
|
||||
if team_member_budget is not None:
|
||||
budget_request.max_budget = team_member_budget
|
||||
if team_member_rpm_limit is not None:
|
||||
budget_request.rpm_limit = team_member_rpm_limit
|
||||
if team_member_tpm_limit is not None:
|
||||
budget_request.tpm_limit = team_member_tpm_limit
|
||||
|
||||
team_member_budget_table = await new_budget(
|
||||
budget_obj=budget_request,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
# Add team_member_budget_id as metadata field to team table
|
||||
if new_team_data_json.get("metadata") is None:
|
||||
new_team_data_json["metadata"] = {}
|
||||
new_team_data_json["metadata"][
|
||||
"team_member_budget_id"
|
||||
] = team_member_budget_table.budget_id
|
||||
|
||||
# Remove team member fields from new_team_data_json
|
||||
TeamMemberBudgetHandler._clean_team_member_fields(new_team_data_json)
|
||||
|
||||
return new_team_data_json
|
||||
|
||||
@staticmethod
|
||||
async def upsert_team_member_budget_table(
|
||||
team_table: LiteLLM_TeamTable,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
updated_kv: dict,
|
||||
team_member_budget: Optional[float] = None,
|
||||
team_member_rpm_limit: Optional[int] = None,
|
||||
team_member_tpm_limit: Optional[int] = None,
|
||||
) -> dict:
|
||||
"""Upsert team member budget table with provided limits"""
|
||||
from litellm.proxy._types import BudgetNewRequest
|
||||
from litellm.proxy.management_endpoints.budget_management_endpoints import (
|
||||
update_budget,
|
||||
)
|
||||
|
||||
if team_table.metadata is None:
|
||||
team_table.metadata = {}
|
||||
|
||||
team_member_budget_id = team_table.metadata.get("team_member_budget_id")
|
||||
if team_member_budget_id is not None and isinstance(team_member_budget_id, str):
|
||||
# Budget exists - create update request with only provided values
|
||||
budget_request = BudgetNewRequest(budget_id=team_member_budget_id)
|
||||
|
||||
if team_member_budget is not None:
|
||||
budget_request.max_budget = team_member_budget
|
||||
if team_member_rpm_limit is not None:
|
||||
budget_request.rpm_limit = team_member_rpm_limit
|
||||
if team_member_tpm_limit is not None:
|
||||
budget_request.tpm_limit = team_member_tpm_limit
|
||||
|
||||
budget_row = await update_budget(
|
||||
budget_obj=budget_request,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
verbose_proxy_logger.info(
|
||||
f"Updated team member budget table: {budget_row.budget_id}, with team_member_budget={team_member_budget}, team_member_rpm_limit={team_member_rpm_limit}, team_member_tpm_limit={team_member_tpm_limit}"
|
||||
)
|
||||
if updated_kv.get("metadata") is None:
|
||||
updated_kv["metadata"] = {}
|
||||
updated_kv["metadata"]["team_member_budget_id"] = budget_row.budget_id
|
||||
|
||||
else: # budget does not exist
|
||||
updated_kv = await TeamMemberBudgetHandler.create_team_member_budget_table(
|
||||
data=team_table,
|
||||
new_team_data_json=updated_kv,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
team_member_budget=team_member_budget,
|
||||
team_member_rpm_limit=team_member_rpm_limit,
|
||||
team_member_tpm_limit=team_member_tpm_limit,
|
||||
)
|
||||
|
||||
# Remove team member fields from updated_kv
|
||||
TeamMemberBudgetHandler._clean_team_member_fields(updated_kv)
|
||||
return updated_kv
|
||||
|
||||
@staticmethod
|
||||
def _clean_team_member_fields(data_dict: dict) -> None:
|
||||
"""Remove team member fields from data dictionary"""
|
||||
data_dict.pop("team_member_budget", None)
|
||||
data_dict.pop("team_member_rpm_limit", None)
|
||||
data_dict.pop("team_member_tpm_limit", None)
|
||||
|
||||
|
||||
def _is_available_team(team_id: str, user_api_key_dict: UserAPIKeyAuth) -> bool:
|
||||
if litellm.default_internal_user_params is None:
|
||||
return False
|
||||
|
|
@ -136,93 +267,6 @@ async def get_all_team_memberships(
|
|||
return returned_tm
|
||||
|
||||
|
||||
async def _create_team_member_budget_table(
|
||||
data: Union[NewTeamRequest, LiteLLM_TeamTable],
|
||||
new_team_data_json: dict,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
team_member_budget: float,
|
||||
) -> dict:
|
||||
"""Allows admin to create 1 budget, that applies to all team members"""
|
||||
from litellm.proxy._types import BudgetNewRequest
|
||||
from litellm.proxy.management_endpoints.budget_management_endpoints import (
|
||||
new_budget,
|
||||
)
|
||||
|
||||
if data.team_alias is not None:
|
||||
budget_id = (
|
||||
f"team-{data.team_alias.replace(' ', '-')}-budget-{uuid.uuid4().hex}"
|
||||
)
|
||||
else:
|
||||
budget_id = f"team-budget-{uuid.uuid4().hex}"
|
||||
|
||||
team_member_budget_table = await new_budget(
|
||||
budget_obj=BudgetNewRequest(
|
||||
max_budget=team_member_budget,
|
||||
budget_duration=data.budget_duration,
|
||||
budget_id=budget_id,
|
||||
),
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
|
||||
# Add team_member_budget_id as metadata field to team table
|
||||
if new_team_data_json.get("metadata") is None:
|
||||
new_team_data_json["metadata"] = {}
|
||||
new_team_data_json["metadata"][
|
||||
"team_member_budget_id"
|
||||
] = team_member_budget_table.budget_id
|
||||
new_team_data_json.pop(
|
||||
"team_member_budget", None
|
||||
) # remove team_member_budget from new_team_data_json
|
||||
|
||||
return new_team_data_json
|
||||
|
||||
|
||||
async def _upsert_team_member_budget_table(
|
||||
team_table: LiteLLM_TeamTable,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
team_member_budget: float,
|
||||
updated_kv: dict,
|
||||
) -> dict:
|
||||
"""
|
||||
Add budget if none exists
|
||||
|
||||
If budget exists, update it
|
||||
"""
|
||||
from litellm.proxy._types import BudgetNewRequest
|
||||
from litellm.proxy.management_endpoints.budget_management_endpoints import (
|
||||
update_budget,
|
||||
)
|
||||
|
||||
if team_table.metadata is None:
|
||||
team_table.metadata = {}
|
||||
|
||||
team_member_budget_id = team_table.metadata.get("team_member_budget_id")
|
||||
if team_member_budget_id is not None and isinstance(team_member_budget_id, str):
|
||||
# Budget exists
|
||||
budget_row = await update_budget(
|
||||
budget_obj=BudgetNewRequest(
|
||||
budget_id=team_member_budget_id,
|
||||
max_budget=team_member_budget,
|
||||
),
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
verbose_proxy_logger.info(
|
||||
f"Updated team member budget table: {budget_row.budget_id}, with team_member_budget={team_member_budget}"
|
||||
)
|
||||
if updated_kv.get("metadata") is None:
|
||||
updated_kv["metadata"] = {}
|
||||
updated_kv["metadata"]["team_member_budget_id"] = budget_row.budget_id
|
||||
|
||||
else: # budget does not exist
|
||||
updated_kv = await _create_team_member_budget_table(
|
||||
data=team_table,
|
||||
new_team_data_json=updated_kv,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
team_member_budget=team_member_budget,
|
||||
)
|
||||
updated_kv.pop("team_member_budget", None)
|
||||
return updated_kv
|
||||
|
||||
|
||||
#### TEAM MANAGEMENT ####
|
||||
@router.post(
|
||||
|
|
@ -268,6 +312,8 @@ async def new_team( # noqa: PLR0915
|
|||
- prompts: Optional[List[str]] - List of prompts that the team is allowed to use.
|
||||
- object_permission: Optional[LiteLLM_ObjectPermissionBase] - team-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"]}. IF null or {} then no object permission.
|
||||
- team_member_budget: Optional[float] - The maximum budget allocated to an individual team member.
|
||||
- team_member_rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for individual team members.
|
||||
- team_member_tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for individual team members.
|
||||
- team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo"
|
||||
- prompts: Optional[List[str]] - List of allowed prompts for the team. If specified, the team will only be able to use these specific prompts.
|
||||
|
||||
|
|
@ -421,12 +467,18 @@ async def new_team( # noqa: PLR0915
|
|||
|
||||
## Create Team Member Budget Table
|
||||
data_json = data.json()
|
||||
if data.team_member_budget is not None:
|
||||
data_json = await _create_team_member_budget_table(
|
||||
if TeamMemberBudgetHandler.should_create_budget(
|
||||
team_member_budget=data.team_member_budget,
|
||||
team_member_rpm_limit=data.team_member_rpm_limit,
|
||||
team_member_tpm_limit=data.team_member_tpm_limit,
|
||||
):
|
||||
data_json = await TeamMemberBudgetHandler.create_team_member_budget_table(
|
||||
data=data,
|
||||
new_team_data_json=data_json,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
team_member_budget=data.team_member_budget,
|
||||
team_member_rpm_limit=data.team_member_rpm_limit,
|
||||
team_member_tpm_limit=data.team_member_tpm_limit,
|
||||
)
|
||||
|
||||
## ADD TO TEAM TABLE
|
||||
|
|
@ -705,6 +757,8 @@ async def update_team(
|
|||
- prompts: Optional[List[str]] - List of prompts that the team is allowed to use.
|
||||
- object_permission: Optional[LiteLLM_ObjectPermissionBase] - team-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"]}. IF null or {} then no object permission.
|
||||
- team_member_budget: Optional[float] - The maximum budget allocated to an individual team member.
|
||||
- team_member_rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for individual team members.
|
||||
- team_member_tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for individual team members.
|
||||
- team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo"
|
||||
Example - update team TPM Limit
|
||||
|
||||
|
|
@ -797,15 +851,21 @@ async def update_team(
|
|||
# set the budget_reset_at in DB
|
||||
updated_kv["budget_reset_at"] = reset_at
|
||||
|
||||
if data.team_member_budget is not None:
|
||||
updated_kv = await _upsert_team_member_budget_table(
|
||||
if TeamMemberBudgetHandler.should_create_budget(
|
||||
team_member_budget=data.team_member_budget,
|
||||
team_member_rpm_limit=data.team_member_rpm_limit,
|
||||
team_member_tpm_limit=data.team_member_tpm_limit,
|
||||
):
|
||||
updated_kv = await TeamMemberBudgetHandler.upsert_team_member_budget_table(
|
||||
team_table=existing_team_row,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
updated_kv=updated_kv,
|
||||
team_member_budget=data.team_member_budget,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
team_member_rpm_limit=data.team_member_rpm_limit,
|
||||
team_member_tpm_limit=data.team_member_tpm_limit,
|
||||
)
|
||||
else:
|
||||
updated_kv.pop("team_member_budget", None)
|
||||
TeamMemberBudgetHandler._clean_team_member_fields(updated_kv)
|
||||
|
||||
# Check object permission
|
||||
if data.object_permission is not None:
|
||||
|
|
@ -1557,6 +1617,8 @@ async def team_member_update(
|
|||
max_budget=data.max_budget_in_team,
|
||||
existing_budget_id=identified_budget_id,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
tpm_limit=data.tpm_limit,
|
||||
rpm_limit=data.rpm_limit,
|
||||
)
|
||||
|
||||
### update team member role
|
||||
|
|
@ -1587,6 +1649,8 @@ async def team_member_update(
|
|||
user_id=received_user_id,
|
||||
user_email=data.user_email,
|
||||
max_budget_in_team=data.max_budget_in_team,
|
||||
tpm_limit=data.tpm_limit,
|
||||
rpm_limit=data.rpm_limit,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -96,7 +96,6 @@ class AnthropicPassthroughLoggingHandler:
|
|||
handles streaming and non-streaming responses
|
||||
"""
|
||||
try:
|
||||
|
||||
response_cost = litellm.completion_cost(
|
||||
completion_response=litellm_model_response,
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
model_list:
|
||||
- model_name: vertex_ai/*
|
||||
- model_name: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0
|
||||
litellm_params:
|
||||
model: vertex_ai/*
|
||||
model: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0
|
||||
- model_name: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0
|
||||
litellm_params:
|
||||
model: bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0
|
||||
|
|
|
|||
|
|
@ -4875,7 +4875,9 @@ async def websocket_endpoint(
|
|||
await websocket.accept()
|
||||
|
||||
# Only use explicit parameters, not all query params
|
||||
query_params: RealtimeQueryParams = {"model": model, "intent": intent}
|
||||
query_params: RealtimeQueryParams = {"model": model}
|
||||
if intent is not None:
|
||||
query_params["intent"] = intent
|
||||
|
||||
data = {
|
||||
"model": model,
|
||||
|
|
|
|||
|
|
@ -173,7 +173,7 @@ async def _realtime_health_check(
|
|||
)
|
||||
elif custom_llm_provider == "openai":
|
||||
url = openai_realtime._construct_url(
|
||||
api_base=api_base or "https://api.openai.com/", query_params=RealtimeQueryParams(model=model)
|
||||
api_base=api_base or "https://api.openai.com/", query_params={"model": model}
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported model: {model}")
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import base64
|
||||
from typing import Any, Dict, Optional, Union, cast, get_type_hints
|
||||
from typing import Any, Dict, Optional, Union, cast, get_type_hints, overload
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -95,28 +95,57 @@ class ResponsesAPIRequestUtils:
|
|||
)
|
||||
|
||||
return cast(ResponsesAPIOptionalRequestParams, filtered_params)
|
||||
|
||||
|
||||
@overload
|
||||
@staticmethod
|
||||
def _update_responses_api_response_id_with_model_id(
|
||||
responses_api_response: ResponsesAPIResponse,
|
||||
custom_llm_provider: Optional[str],
|
||||
litellm_metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> ResponsesAPIResponse:
|
||||
"""
|
||||
Update the responses_api_response_id with model_id and custom_llm_provider
|
||||
...
|
||||
|
||||
This builds a composite ID containing the custom LLM provider, model ID, and original response ID
|
||||
@overload
|
||||
@staticmethod
|
||||
def _update_responses_api_response_id_with_model_id(
|
||||
responses_api_response: Dict[str, Any],
|
||||
custom_llm_provider: Optional[str],
|
||||
litellm_metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
...
|
||||
|
||||
@staticmethod
|
||||
def _update_responses_api_response_id_with_model_id(
|
||||
responses_api_response: Union[ResponsesAPIResponse, Dict[str, Any]],
|
||||
custom_llm_provider: Optional[str],
|
||||
litellm_metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> Union[ResponsesAPIResponse, Dict[str, Any]]:
|
||||
"""Update the responses_api_response_id with model_id and custom_llm_provider.
|
||||
|
||||
Handles both ``ResponsesAPIResponse`` objects and plain dictionaries returned
|
||||
by some streaming providers.
|
||||
"""
|
||||
litellm_metadata = litellm_metadata or {}
|
||||
model_info: Dict[str, Any] = litellm_metadata.get("model_info", {}) or {}
|
||||
model_id = model_info.get("id")
|
||||
|
||||
# access the response id based on the object type
|
||||
response_id = (
|
||||
responses_api_response["id"]
|
||||
if isinstance(responses_api_response, dict)
|
||||
else responses_api_response.id
|
||||
)
|
||||
|
||||
updated_id = ResponsesAPIRequestUtils._build_responses_api_response_id(
|
||||
model_id=model_id,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
response_id=responses_api_response.id,
|
||||
response_id=response_id,
|
||||
)
|
||||
|
||||
responses_api_response.id = updated_id
|
||||
if isinstance(responses_api_response, dict):
|
||||
responses_api_response["id"] = updated_id
|
||||
else:
|
||||
responses_api_response.id = updated_id
|
||||
return responses_api_response
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -4322,8 +4322,10 @@ class Router:
|
|||
deployment_name = kwargs["litellm_params"]["metadata"].get(
|
||||
"deployment", None
|
||||
) # stable name - works for wildcard routes as well
|
||||
model_group = standard_logging_object.get("model_group", None)
|
||||
id = standard_logging_object.get("model_id", None)
|
||||
# Get model_group and id from kwargs like the sync version does
|
||||
model_group = kwargs["litellm_params"]["metadata"].get("model_group", None)
|
||||
model_info = kwargs["litellm_params"].get("model_info", {}) or {}
|
||||
id = model_info.get("id", None)
|
||||
if model_group is None or id is None:
|
||||
return
|
||||
elif isinstance(id, int):
|
||||
|
|
@ -4386,7 +4388,6 @@ class Router:
|
|||
# Update usage
|
||||
# ------------
|
||||
# update cache
|
||||
|
||||
pipeline_operations: List[RedisPipelineIncrementOperation] = []
|
||||
|
||||
## TPM
|
||||
|
|
@ -5475,6 +5476,11 @@ class Router:
|
|||
pass
|
||||
|
||||
## GET LITELLM MODEL INFO - raises exception, if model is not mapped
|
||||
if model is None:
|
||||
# Handle case where base_model is None (e.g., Azure models without base_model set)
|
||||
# Use the original model from litellm_params
|
||||
model = _model
|
||||
|
||||
if not model.startswith("{}/".format(custom_llm_provider)):
|
||||
model_info_name = "{}/{}".format(custom_llm_provider, model)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import random
|
|||
from typing import TYPE_CHECKING, Any, Dict, List, Union
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.litellm_core_utils.core_helpers import safe_divide
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router as _Router
|
||||
|
|
@ -46,7 +47,7 @@ def simple_shuffle(
|
|||
weights = [m["litellm_params"].get("weight", 0) for m in healthy_deployments]
|
||||
verbose_router_logger.debug(f"\nweight {weights}")
|
||||
total_weight = sum(weights)
|
||||
weights = [weight / total_weight for weight in weights]
|
||||
weights = [safe_divide(weight, total_weight, 0) for weight in weights]
|
||||
verbose_router_logger.debug(f"\n weights {weights}")
|
||||
# Perform weighted random pick
|
||||
selected_index = random.choices(range(len(weights)), weights=weights)[0]
|
||||
|
|
@ -63,7 +64,7 @@ def simple_shuffle(
|
|||
rpms = [m["litellm_params"].get("rpm", 0) for m in healthy_deployments]
|
||||
verbose_router_logger.debug(f"\nrpms {rpms}")
|
||||
total_rpm = sum(rpms)
|
||||
weights = [rpm / total_rpm for rpm in rpms]
|
||||
weights = [safe_divide(rpm, total_rpm, 0) for rpm in rpms]
|
||||
verbose_router_logger.debug(f"\n weights {weights}")
|
||||
# Perform weighted random pick
|
||||
selected_index = random.choices(range(len(rpms)), weights=weights)[0]
|
||||
|
|
@ -80,7 +81,7 @@ def simple_shuffle(
|
|||
tpms = [m["litellm_params"].get("tpm", 0) for m in healthy_deployments]
|
||||
verbose_router_logger.debug(f"\ntpms {tpms}")
|
||||
total_tpm = sum(tpms)
|
||||
weights = [tpm / total_tpm for tpm in tpms]
|
||||
weights = [safe_divide(tpm, total_tpm, 0) for tpm in tpms]
|
||||
verbose_router_logger.debug(f"\n weights {weights}")
|
||||
# Perform weighted random pick
|
||||
selected_index = random.choices(range(len(tpms)), weights=weights)[0]
|
||||
|
|
|
|||
|
|
@ -25,8 +25,8 @@ class LangfuseSpanAttributes(str, Enum):
|
|||
MASK_OUTPUT = "langfuse.generation.mask_output"
|
||||
|
||||
# ---- Trace-level metadata ----
|
||||
TRACE_USER_ID = "langfuse.trace.user_id"
|
||||
SESSION_ID = "langfuse.trace.session_id"
|
||||
TRACE_USER_ID = "user.id"
|
||||
SESSION_ID = "session.id"
|
||||
TAGS = "langfuse.trace.tags"
|
||||
TRACE_NAME = "langfuse.trace.name"
|
||||
TRACE_ID = "langfuse.trace.id"
|
||||
|
|
|
|||
|
|
@ -176,6 +176,11 @@ DEFINED_PROMETHEUS_METRICS = Literal[
|
|||
"litellm_deployment_failure_responses",
|
||||
"litellm_deployment_total_requests",
|
||||
"litellm_deployment_success_responses",
|
||||
"litellm_pod_lock_manager_size",
|
||||
"litellm_in_memory_daily_spend_update_queue_size",
|
||||
"litellm_redis_daily_spend_update_queue_size",
|
||||
"litellm_in_memory_spend_update_queue_size",
|
||||
"litellm_redis_spend_update_queue_size",
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -378,6 +383,17 @@ class PrometheusMetricLabels:
|
|||
|
||||
litellm_deployment_success_responses = litellm_deployment_total_requests
|
||||
|
||||
# Buffer monitoring metrics - these typically don't need additional labels
|
||||
litellm_pod_lock_manager_size: List[str] = []
|
||||
|
||||
litellm_in_memory_daily_spend_update_queue_size: List[str] = []
|
||||
|
||||
litellm_redis_daily_spend_update_queue_size: List[str] = []
|
||||
|
||||
litellm_in_memory_spend_update_queue_size: List[str] = []
|
||||
|
||||
litellm_redis_spend_update_queue_size: List[str] = []
|
||||
|
||||
@staticmethod
|
||||
def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> List[str]:
|
||||
default_labels = getattr(PrometheusMetricLabels, label_name)
|
||||
|
|
|
|||
|
|
@ -3615,7 +3615,7 @@ def get_optional_params( # noqa: PLR0915
|
|||
elif "anthropic" in bedrock_base_model and bedrock_route == "invoke":
|
||||
if bedrock_base_model.startswith("anthropic.claude-3"):
|
||||
optional_params = (
|
||||
litellm.AmazonAnthropicClaude3Config().map_openai_params(
|
||||
litellm.AmazonAnthropicClaudeConfig().map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
|
|
@ -6975,7 +6975,7 @@ class ProviderConfigManager:
|
|||
):
|
||||
return litellm.AmazonAnthropicConfig()
|
||||
else:
|
||||
return litellm.AmazonAnthropicClaude3Config()
|
||||
return litellm.AmazonAnthropicClaudeConfig()
|
||||
elif (
|
||||
bedrock_invoke_provider == "meta" or bedrock_invoke_provider == "llama"
|
||||
): # amazon / meta llms
|
||||
|
|
@ -7071,7 +7071,8 @@ class ProviderConfigManager:
|
|||
# The 'BEDROCK' provider corresponds to Amazon's implementation of Anthropic Claude v3.
|
||||
# This mapping ensures that the correct configuration is returned for BEDROCK.
|
||||
elif litellm.LlmProviders.BEDROCK == provider:
|
||||
return litellm.AmazonAnthropicClaude3MessagesConfig()
|
||||
from litellm.llms.bedrock.common_utils import BedrockModelInfo
|
||||
return BedrockModelInfo.get_bedrock_provider_config_for_messages_api(model)
|
||||
elif litellm.LlmProviders.VERTEX_AI == provider:
|
||||
if "claude" in model:
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import (
|
||||
|
|
@ -7280,6 +7281,12 @@ class ProviderConfigManager:
|
|||
)
|
||||
|
||||
return get_azure_image_generation_config(model)
|
||||
elif LlmProviders.AZURE_AI == provider:
|
||||
from litellm.llms.azure_ai.image_generation import (
|
||||
get_azure_ai_image_generation_config,
|
||||
)
|
||||
|
||||
return get_azure_ai_image_generation_config(model)
|
||||
elif LlmProviders.XINFERENCE == provider:
|
||||
from litellm.llms.xinference.image_generation import (
|
||||
get_xinference_image_generation_config,
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue