scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-06 08:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge branch 'BerriAI:main' into main

Browse source
This commit is contained in:
Franklin 2025-09-24 17:12:23 +08:00 • committed by GitHub
commit 2495aa50f1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
243 changed files with 15760 additions and 4099 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
.circleci/config.yml
View file

@ -1050,6 +1050,51 @@ jobs:
ls
python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml litellm_mapped_tests_coverage.xml
mv .coverage litellm_mapped_tests_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- litellm_mapped_tests_coverage.xml
- litellm_mapped_tests_coverage
litellm_mapped_enterprise_tests:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest-mock==3.12.0"
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "hypercorn==0.17.3"
pip install "pydantic==2.10.2"
pip install "mcp==1.10.1"
pip install "requests-mock>=1.12.1"
pip install "responses==0.25.7"
pip install "pytest-xdist==3.6.1"
pip install "semantic_router==0.1.10"
pip install "fastapi-offline==1.7.3"
- setup_litellm_enterprise_pip
- run:
name: Run enterprise tests
command: |
@ -1476,6 +1521,7 @@ jobs:
- run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
- run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
- run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
- run: python ./tests/code_coverage_tests/check_fastuuid_usage.py
- run: helm lint ./deploy/charts/litellm-helm
db_migration_disable_update_check:
@ -1779,8 +1825,8 @@ jobs:
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e AZURE_API_KEY=$AZURE_BATCHES_API_KEY \
-e AZURE_API_BASE=$AZURE_BATCHES_API_BASE \
-e AZURE_API_KEY=$AZURE_API_KEY \
-e AZURE_API_BASE=$AZURE_API_BASE \
-e AZURE_API_VERSION="2024-05-01-preview" \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
@ -3175,6 +3221,12 @@ workflows:
only:
- main
- /litellm_.*/
- litellm_mapped_enterprise_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- litellm_mapped_tests:
filters:
branches:
@ -3219,6 +3271,7 @@ workflows:
- guardrails_testing
- llm_responses_api_testing
- litellm_mapped_tests
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing
@ -3279,6 +3332,7 @@ workflows:
- google_generate_content_endpoint_testing
- llm_responses_api_testing
- litellm_mapped_tests
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing

48
.github/workflows/test-mcp.yml vendored Normal file
View file

@ -0,0 +1,48 @@
name: LiteLLM MCP Tests (folder - tests/mcp_tests)
on:
pull_request:
branches: [ main ]
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- name: Thank You Message
run: |
echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12'
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Install dependencies
run: |
poetry install --with dev,proxy-dev --extras "proxy semantic-router"
poetry run pip install "pytest==7.3.1"
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install "pytest-cov==5.0.0"
poetry run pip install "pytest-asyncio==0.21.1"
poetry run pip install "respx==0.22.0"
poetry run pip install "pydantic==2.10.2"
poetry run pip install "mcp==1.10.1"
poetry run pip install pytest-xdist
- name: Setup litellm-enterprise as local package
run: |
cd enterprise
python -m pip install -e .
cd ..
- name: Run MCP tests
run: |
poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5

3
Dockerfile
View file

@ -41,9 +41,6 @@ RUN pip uninstall jwt -y
RUN pip uninstall PyJWT -y
RUN pip install PyJWT==2.9.0 --no-cache-dir
# Build Admin UI
RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE AS runtime

62
cookbook/litellm_proxy_server/cli_token_usage.py Normal file
View file

@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""
Example: Using CLI token with LiteLLM SDK
This example shows how to use the CLI authentication token
in your Python scripts after running `litellm-proxy login`.
"""
from textwrap import indent
import litellm
LITELLM_BASE_URL = "http://localhost:4000/"
def main():
"""Using CLI token with LiteLLM SDK"""
print("🚀 Using CLI Token with LiteLLM SDK")
print("=" * 40)
#litellm._turn_on_debug()
# Get the CLI token
api_key = litellm.get_litellm_gateway_api_key()
if not api_key:
print("❌ No CLI token found. Please run 'litellm-proxy login' first.")
return
print("✅ Found CLI token.")
available_models = litellm.get_valid_models(
check_provider_endpoint=True,
custom_llm_provider="litellm_proxy",
api_key=api_key,
api_base=LITELLM_BASE_URL
)
print("✅ Available models:")
if available_models:
for i, model in enumerate(available_models, 1):
print(f" {i:2d}. {model}")
else:
print(" No models available")
# Use with LiteLLM
try:
response = litellm.completion(
model="litellm_proxy/gemini/gemini-2.5-flash",
messages=[{"role": "user", "content": "Hello from CLI token!"}],
api_key=api_key,
base_url=LITELLM_BASE_URL
)
print(f"✅ LLM Response: {response.model_dump_json(indent=4)}")
except Exception as e:
print(f"❌ Error: {e}")
if __name__ == "__main__":
main()
print("\n💡 Tips:")
print("1. Run 'litellm-proxy login' to authenticate first")
print("2. Replace 'https://your-proxy.com' with your actual proxy URL")
print("3. The token is stored locally at ~/.litellm/token.json")

134
cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md
View file

@ -4,10 +4,10 @@ This document provides comprehensive instructions for AI agents to generate rele
## Required Inputs
1. **Release Version** (e.g., `v1.76.3-stable`)
1. **Release Version** (e.g., `v1.77.3-stable`)
2. **PR Diff/Changelog** - List of PRs with titles and contributors
3. **Previous Version Commit Hash** - To compare model pricing changes
4. **Reference Release Notes** - Previous release notes to follow style/format
4. **Reference Release Notes** - Use recent stable releases (v1.76.3-stable, v1.77.2-stable) as templates for consistent formatting
## Step-by-Step Process
@ -26,12 +26,12 @@ git diff <previous_commit_hash> HEAD -- model_prices_and_context_window.json
### 2. Release Notes Structure
Follow this exact structure based on `docs/my-website/release_notes/v1.76.1-stable/index.md`:
Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable):
```markdown
---
title: "v1.76.X-stable - [Key Theme]"
slug: "v1-76-X"
title: "v1.77.X-stable - [Key Theme]"
slug: "v1-77-X"
date: YYYY-MM-DDTHH:mm:ss
authors: [standard author block]
hide_table_of_contents: false
@ -43,23 +43,42 @@ hide_table_of_contents: false
## Key Highlights
[3-5 bullet points of major features]
## Major Changes
[Critical changes users need to know]
## Performance Improvements
[Performance-related changes]
## New Models / Updated Models
[Detailed model tables and provider updates]
#### New Model Support
[Model pricing table]
#### Features
[Provider-specific features organized by provider]
### Bug Fixes
[Provider-specific bug fixes organized by provider]
#### New Provider Support
[New provider integrations]
## LLM API Endpoints
[API-related features and fixes]
#### Features
[API-specific features organized by API type]
#### Bugs
[General bug fixes]
## Management Endpoints / UI
[Admin interface and management changes]
#### Features
[UI and management features]
#### Bugs
[Management-related bug fixes]
## Logging / Guardrail Integrations
[Observability and security features]
#### Features
[Organized by integration provider with proper doc links]
#### Guardrails
[Guardrail-specific features and fixes]
#### New Integration
[Major new integrations]
## Performance / Loadbalancing / Reliability improvements
[Infrastructure improvements]
@ -86,21 +105,27 @@ hide_table_of_contents: false
**New Models/Updated Models:**
- Extract from model_prices_and_context_window.json diff
- Create tables with: Provider, Model, Context Window, Input Cost, Output Cost, Features
- Group by provider
- Note pricing corrections
- Highlight deprecated models
- **Structure:**
- `#### New Model Support` - pricing table
- `#### Features` - organized by provider with documentation links
- `### Bug Fixes` - provider-specific bug fixes
- `#### New Provider Support` - major new provider integrations
- Group by provider with proper doc links: `**[Provider Name](../../docs/providers/[provider])**`
- Use bullet points under each provider for multiple features
- Separate features from bug fixes clearly
**Provider Features:**
- Group by provider (Gemini, OpenAI, Anthropic, etc.)
- Link to provider docs: `../../docs/providers/[provider_name]`
- Separate features from bug fixes
**API Endpoints:**
- Images API
- Video Generation (if applicable)
- Responses API
- Passthrough endpoints
- General chat completions
**LLM API Endpoints:**
- **Structure:**
- `#### Features` - organized by API type (Responses API, Batch API, etc.)
- `#### Bugs` - general bug fixes under **General** category
- **API Categories:**
- Responses API
- Batch API
- CountTokens API
- Images API
- Video Generation (if applicable)
- General (miscellaneous improvements)
- Use proper documentation links for each API type
**UI/Management:**
- Authentication changes
@ -108,11 +133,19 @@ hide_table_of_contents: false
- Team management
- Key management
**Integrations:**
- Logging providers (Datadog, Braintrust, etc.)
- Guardrails
- Cost tracking
- Observability
**Logging / Guardrail Integrations:**
- **Structure:**
- `#### Features` - organized by integration provider with proper doc links
- `#### Guardrails` - guardrail-specific features and fixes
- `#### New Integration` - major new integrations
- **Integration Categories:**
- **[DataDog](../../docs/proxy/logging#datadog)** - group all DataDog-related changes
- **[Langfuse](../../docs/proxy/logging#langfuse)** - Langfuse-specific features
- **[Prometheus](../../docs/proxy/logging#prometheus)** - monitoring improvements
- **[PostHog](../../docs/observability/posthog)** - observability integration
- Other logging providers with proper doc links
- Use bullet points under each provider for multiple features
- Separate logging features from guardrails clearly
### 4. Documentation Linking Strategy
@ -211,10 +244,41 @@ This release has a known issue...
:::
```
**Provider Features:**
**Provider Features (New Models / Updated Models section):**
```markdown
#### Features
- **[Provider Name](../../docs/providers/provider)**
- Feature description - [PR #XXXXX](link)
- Another feature description - [PR #YYYYY](link)
```
**API Features (LLM API Endpoints section):**
```markdown
#### Features
- **[API Name](../../docs/api_path)**
- Feature description - [PR #XXXXX](link)
- Another feature - [PR #YYYYY](link)
- **General**
- Miscellaneous improvements - [PR #ZZZZZ](link)
```
**Integration Features (Logging / Guardrail Integrations section):**
```markdown
#### Features
- **[Integration Name](../../docs/proxy/logging#integration)**
- Feature description - [PR #XXXXX](link)
- Bug fix description - [PR #YYYYY](link)
```
**Bug Fixes Pattern:**
```markdown
### Bug Fixes
- **[Provider/Component Name](../../docs/providers/provider)**
- Bug fix description - [PR #XXXXX](link)
```
### 10. Missing Documentation Check

58
docs/my-website/docs/completion/provider_specific_params.md
View file

@ -423,7 +423,7 @@ model_list:
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
-d '{
"model": "llama-3-8b-instruct",
"messages": [
{
@ -431,6 +431,56 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
"content": "What'\''s the weather like in Boston today?"
}
],
"adapater_id": "my-special-adapter-id" # 👈 PROVIDER-SPECIFIC PARAM
}'
```
"adapater_id": "my-special-adapter-id"
}'
```
## Provider-Specific Metadata Parameters
| Provider | Parameter | Use Case |
|----------|-----------|----------|
| **AWS Bedrock** | `requestMetadata` | Cost attribution, logging |
| **Gemini/Vertex AI** | `labels` | Resource labeling |
| **Anthropic** | `metadata` | User identification |
<Tabs>
<TabItem value="bedrock" label="AWS Bedrock">
```python
import litellm
response = litellm.completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
messages=[{"role": "user", "content": "Hello!"}],
requestMetadata={"cost_center": "engineering"}
)
```
</TabItem>
<TabItem value="gemini" label="Gemini/Vertex AI">
```python
import litellm
response = litellm.completion(
model="vertex_ai/gemini-pro",
messages=[{"role": "user", "content": "Hello!"}],
labels={"environment": "production"}
)
```
</TabItem>
<TabItem value="anthropic" label="Anthropic">
```python
import litellm
response = litellm.completion(
model="anthropic/claude-3-sonnet-20240229",
messages=[{"role": "user", "content": "Hello!"}],
metadata={"user_id": "user123"}
)
```
</TabItem>
</Tabs>

213
docs/my-website/docs/completion/shared_session.md Normal file
View file

@ -0,0 +1,213 @@
# Shared Session Support
## Overview
LiteLLM now supports sharing `aiohttp.ClientSession` instances across multiple API calls to avoid creating unnecessary new sessions. This improves performance and resource utilization.
## Usage
### Basic Usage
```python
import asyncio
from aiohttp import ClientSession
from litellm import acompletion
async def main():
# Create a shared session
async with ClientSession() as shared_session:
# Use the same session for multiple calls
response1 = await acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}],
shared_session=shared_session
)
response2 = await acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": "How are you?"}],
shared_session=shared_session
)
# Both calls reuse the same session!
asyncio.run(main())
```
### Without Shared Session (Default)
```python
import asyncio
from litellm import acompletion
async def main():
# Each call creates a new session
response1 = await acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}]
)
response2 = await acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": "How are you?"}]
)
# Two separate sessions created
asyncio.run(main())
```
## Benefits
- **Performance**: Reuse HTTP connections across multiple calls
- **Resource Efficiency**: Reduce memory and connection overhead
- **Better Control**: Manage session lifecycle explicitly
- **Debugging**: Easy to trace which calls use which sessions
## Debug Logging
Enable debug logging to see session reuse in action:
```python
import os
import litellm
# Enable debug logging
os.environ['LITELLM_LOG'] = 'DEBUG'
# You'll see logs like:
# 🔄 SHARED SESSION: acompletion called with shared_session (ID: 12345)
# ✅ SHARED SESSION: Reusing existing ClientSession (ID: 12345)
```
## Common Patterns
### FastAPI Integration
```python
from fastapi import FastAPI
import aiohttp
import litellm
app = FastAPI()
@app.post("/chat")
async def chat(messages: list[dict]):
# Create session per request
async with aiohttp.ClientSession() as session:
return await litellm.acompletion(
model="gpt-4o",
messages=messages,
shared_session=session
)
```
### Batch Processing
```python
import asyncio
from aiohttp import ClientSession
from litellm import acompletion
async def process_batch(messages_list):
async with ClientSession() as shared_session:
tasks = []
for messages in messages_list:
task = acompletion(
model="gpt-4o",
messages=messages,
shared_session=shared_session
)
tasks.append(task)
# All tasks use the same session
results = await asyncio.gather(*tasks)
return results
```
### Custom Session Configuration
```python
import aiohttp
import litellm
# Create optimized session
async with aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=180),
connector=aiohttp.TCPConnector(limit=300, limit_per_host=75)
) as shared_session:
response = await litellm.acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}],
shared_session=shared_session
)
```
## Implementation Details
The `shared_session` parameter is threaded through the entire LiteLLM call chain:
1. **`acompletion()`** - Accepts `shared_session` parameter
2. **`BaseLLMHTTPHandler`** - Passes session to HTTP client creation
3. **`AsyncHTTPHandler`** - Uses existing session if provided
4. **`LiteLLMAiohttpTransport`** - Reuses the session for HTTP requests
## Backward Compatibility
- **100% backward compatible** - Existing code works unchanged
- **Optional parameter** - `shared_session=None` by default
- **No breaking changes** - All existing functionality preserved
## Testing
Test the shared session functionality:
```python
import asyncio
from aiohttp import ClientSession
from litellm import acompletion
async def test_shared_session():
async with ClientSession() as session:
print(f"✅ Created session: {id(session)}")
try:
response = await acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": "Hello"}],
shared_session=session,
api_key="your-api-key"
)
print(f"Response: {response.choices[0].message.content}")
except Exception as e:
print(f"✅ Expected error: {type(e).__name__}")
print("✅ Session control working!")
asyncio.run(test_shared_session())
```
## Files Modified
The shared session functionality was added to these files:
- `litellm/main.py` - Added `shared_session` parameter to `acompletion()` and `completion()`
- `litellm/llms/custom_httpx/http_handler.py` - Core session reuse logic
- `litellm/llms/custom_httpx/llm_http_handler.py` - HTTP handler integration
- `litellm/llms/openai/openai.py` - OpenAI provider integration
- `litellm/llms/openai/common_utils.py` - OpenAI client creation
- `litellm/llms/azure/chat/o_series_handler.py` - Azure O Series handler
## Troubleshooting
### Session Not Being Reused
1. **Check debug logs**: Enable `LITELLM_LOG=DEBUG` to see session reuse messages
2. **Verify session is not closed**: Ensure the session is still active when making calls
3. **Check parameter passing**: Make sure `shared_session` is passed to all `acompletion()` calls
### Performance Issues
1. **Session configuration**: Tune `aiohttp.ClientSession` parameters for your use case
2. **Connection limits**: Adjust `limit` and `limit_per_host` in `TCPConnector`
3. **Timeout settings**: Configure appropriate timeouts for your environment

13
docs/my-website/docs/integrations/index.md
View file

@ -2,4 +2,17 @@
This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK).
## AI Agent Frameworks
- **[Letta](./letta.md)** - Build stateful LLM agents with persistent memory using LiteLLM Proxy
## Development Tools
- **[OpenWebUI](../tutorials/openweb_ui.md)** - Self-hosted ChatGPT-style interface
## Observability & Monitoring
- **[Langfuse](../observability/langfuse_integration.md)** - LLM observability and analytics
- **[Prometheus](../proxy/prometheus.md)** - Metrics collection and monitoring
- **[PagerDuty](../proxy/pagerduty.md)** - Incident response and alerting
- **[Datadog](../observability/datadog.md)**
Click into each section to learn more about the integrations.

928
docs/my-website/docs/integrations/letta.md Normal file
View file

@ -0,0 +1,928 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Letta Integration
[Letta](https://github.com/letta-ai/letta) (formerly MemGPT) is a framework for building stateful LLM agents with persistent memory. This guide shows how to integrate both LiteLLM SDK and LiteLLM Proxy with Letta to leverage multiple LLM providers while building memory-enabled agents.
## What is Letta?
Letta allows you to build LLM agents that can:
- Maintain long-term memory across conversations
- Use function calling for tool interactions
- Handle large context windows efficiently
- Persist agent state and memory
## Prerequisites
```bash
pip install letta litellm
```
## Quick Start
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
### 1. Start LiteLLM Proxy
First, create a configuration file for your LiteLLM proxy:
```yaml
# config.yaml
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY
- model_name: claude-3-sonnet
litellm_params:
model: anthropic/claude-3-sonnet-20240229
api_key: os.environ/ANTHROPIC_API_KEY
- model_name: gpt-3.5-turbo
litellm_params:
model: azure/gpt-35-turbo
api_key: os.environ/AZURE_API_KEY
api_base: os.environ/AZURE_API_BASE
api_version: "2023-07-01-preview"
```
Start the proxy:
```bash
litellm --config config.yaml --port 4000
```
### 2. Configure Letta with LiteLLM Proxy
Configure Letta to use your LiteLLM proxy endpoint:
```python
import letta
from letta import create_client
# Configure Letta to use LiteLLM proxy
client = create_client()
# Configure the LLM endpoint
client.set_default_llm_config(
model="gpt-4", # This should match a model from your LiteLLM config
model_endpoint_type="openai",
model_endpoint="http://localhost:4000", # Your LiteLLM proxy URL
context_window=8192
)
# Configure embedding endpoint (optional)
client.set_default_embedding_config(
embedding_endpoint_type="openai",
embedding_endpoint="http://localhost:4000",
embedding_model="text-embedding-ada-002"
)
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
### 1. Configure LiteLLM SDK
Set up your API keys and configure LiteLLM:
```python
import os
import litellm
# Set your API keys
os.environ["OPENAI_API_KEY"] = "your-openai-key"
os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
# Optional: Configure default settings
litellm.set_verbose = True # For debugging
```
### 2. Create Custom LLM Wrapper for Letta
Create a custom LLM wrapper that uses LiteLLM SDK:
```python
import letta
from letta import create_client
from letta.llm_api.llm_api_base import LLMConfig
import litellm
from typing import List, Dict, Any
class LiteLLMWrapper:
def __init__(self, model: str):
self.model = model
def chat_completions_create(self, messages: List[Dict], **kwargs):
# Use LiteLLM SDK for completion
response = litellm.completion(
model=self.model,
messages=messages,
**kwargs
)
return response
# Configure Letta with custom LiteLLM wrapper
client = create_client()
# Set up LLM configuration using direct SDK integration
llm_config = LLMConfig(
model="gpt-4", # or "claude-3-sonnet", "azure/gpt-35-turbo", etc.
model_endpoint_type="openai",
context_window=8192
)
client.set_default_llm_config(llm_config)
```
</TabItem>
</Tabs>
### 3. Create and Use a Letta Agent
<Tabs>
<TabItem value="proxy" label="Using LiteLLM Proxy">
```python
import letta
from letta import create_client
# Create Letta client
client = create_client()
# Create a new agent
agent_state = client.create_agent(
name="my-assistant",
system="You are a helpful assistant with persistent memory.",
llm_config=client.get_default_llm_config(),
embedding_config=client.get_default_embedding_config()
)
# Send a message to the agent
response = client.user_message(
agent_id=agent_state.id,
message="Hi! My name is Alice and I love reading science fiction books."
)
print(f"Agent response: {response.messages[-1].text}")
# Send another message - the agent will remember previous context
response = client.user_message(
agent_id=agent_state.id,
message="What did I tell you about my interests?"
)
print(f"Agent response: {response.messages[-1].text}")
```
</TabItem>
<TabItem value="sdk" label="Using LiteLLM SDK">
```python
import letta
from letta import create_client
import litellm
import os
# Set up environment variables
os.environ["OPENAI_API_KEY"] = "your-openai-key"
# Create Letta client with LiteLLM integration
client = create_client()
# Create a new agent
agent_state = client.create_agent(
name="my-assistant",
system="You are a helpful assistant with persistent memory.",
llm_config=client.get_default_llm_config(),
embedding_config=client.get_default_embedding_config()
)
# Send a message to the agent
response = client.user_message(
agent_id=agent_state.id,
message="Hi! My name is Alice and I love reading science fiction books."
)
print(f"Agent response: {response.messages[-1].text}")
# Send another message - the agent will remember previous context
response = client.user_message(
agent_id=agent_state.id,
message="What did I tell you about my interests?"
)
print(f"Agent response: {response.messages[-1].text}")
```
</TabItem>
</Tabs>
## Advanced Configuration
### Using Different Models for Different Agents
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
```python
from letta import LLMConfig, EmbeddingConfig
# Create different LLM configurations pointing to your proxy
gpt4_config = LLMConfig(
model="gpt-4",
model_endpoint_type="openai",
model_endpoint="http://localhost:4000",
context_window=8192
)
claude_config = LLMConfig(
model="claude-3-sonnet",
model_endpoint_type="openai", # Using OpenAI-compatible endpoint
model_endpoint="http://localhost:4000",
context_window=200000
)
# Create agents with different configurations
research_agent = client.create_agent(
name="research-agent",
system="You are a research assistant specialized in analysis.",
llm_config=claude_config # Use Claude for research tasks
)
creative_agent = client.create_agent(
name="creative-agent",
system="You are a creative writing assistant.",
llm_config=gpt4_config # Use GPT-4 for creative tasks
)
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
import os
import litellm
from letta import LLMConfig, EmbeddingConfig
# Set up API keys for different providers
os.environ["OPENAI_API_KEY"] = "your-openai-key"
os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
# Create different LLM configurations for direct SDK usage
gpt4_config = LLMConfig(
model="openai/gpt-4", # Using LiteLLM model format
model_endpoint_type="openai",
context_window=8192
)
claude_config = LLMConfig(
model="anthropic/claude-3-sonnet-20240229", # Using LiteLLM model format
model_endpoint_type="openai",
context_window=200000
)
# Create agents with different configurations
research_agent = client.create_agent(
name="research-agent",
system="You are a research assistant specialized in analysis.",
llm_config=claude_config # Use Claude for research tasks
)
creative_agent = client.create_agent(
name="creative-agent",
system="You are a creative writing assistant.",
llm_config=gpt4_config # Use GPT-4 for creative tasks
)
```
</TabItem>
</Tabs>
### Function Calling with Tools
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
```python
# Define custom tools for your agent
def search_web(query: str) -> str:
"""Search the web for information"""
# Your web search implementation
return f"Search results for: {query}"
def save_note(content: str) -> str:
"""Save a note to persistent storage"""
# Your note saving implementation
return f"Note saved: {content}"
# Create agent with tools (using proxy endpoint)
agent_state = client.create_agent(
name="research-assistant",
system="You are a research assistant that can search the web and save notes.",
llm_config=client.get_default_llm_config(),
embedding_config=client.get_default_embedding_config(),
tools=[search_web, save_note]
)
# The agent can now use these tools
response = client.user_message(
agent_id=agent_state.id,
message="Search for recent developments in AI and save important findings."
)
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK">
```python
import litellm
import os
# Set up API keys
os.environ["OPENAI_API_KEY"] = "your-openai-key"
# Define custom tools for your agent
def search_web(query: str) -> str:
"""Search the web for information"""
# Your web search implementation
return f"Search results for: {query}"
def save_note(content: str) -> str:
"""Save a note to persistent storage"""
# Your note saving implementation
return f"Note saved: {content}"
# Create agent with tools (using LiteLLM SDK directly)
agent_state = client.create_agent(
name="research-assistant",
system="You are a research assistant that can search the web and save notes.",
llm_config=LLMConfig(
model="openai/gpt-4", # Direct model specification
model_endpoint_type="openai",
context_window=8192
),
embedding_config=client.get_default_embedding_config(),
tools=[search_web, save_note]
)
# The agent can now use these tools
response = client.user_message(
agent_id=agent_state.id,
message="Search for recent developments in AI and save important findings."
)
```
</TabItem>
</Tabs>
## Authentication
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy Authentication">
If your LiteLLM proxy requires authentication:
```python
import os
from letta import LLMConfig
# Set up authenticated configuration
llm_config = LLMConfig(
model="gpt-4",
model_endpoint_type="openai",
model_endpoint="http://localhost:4000",
model_wrapper="openai",
context_window=8192
)
# If using API keys with your proxy
os.environ["OPENAI_API_KEY"] = "your-litellm-proxy-api-key"
client = create_client()
client.set_default_llm_config(llm_config)
```
For proxy with authentication enabled:
```yaml
# config.yaml with auth
general_settings:
master_key: "your-master-key"
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY
```
```python
# Configure Letta with authenticated proxy
llm_config = LLMConfig(
model="gpt-4",
model_endpoint_type="openai",
model_endpoint="http://localhost:4000",
context_window=8192,
api_key="your-master-key" # Proxy master key
)
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK Authentication">
With LiteLLM SDK, set up your provider API keys directly:
```python
import os
import litellm
# Set up API keys for different providers
os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-api-key"
os.environ["AZURE_API_KEY"] = "your-azure-api-key"
os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com"
os.environ["AZURE_API_VERSION"] = "2023-07-01-preview"
# Optional: Configure default settings
litellm.api_key = os.environ.get("OPENAI_API_KEY") # Default key
litellm.set_verbose = True # For debugging
# Use in Letta configuration
from letta import LLMConfig
llm_config = LLMConfig(
model="openai/gpt-4", # Will use OPENAI_API_KEY automatically
model_endpoint_type="openai",
context_window=8192
)
# Or for Azure
azure_config = LLMConfig(
model="azure/gpt-35-turbo",
model_endpoint_type="openai",
context_window=4096
)
```
</TabItem>
</Tabs>
## Load Balancing and Fallbacks
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy Features">
LiteLLM proxy's load balancing and fallback features work seamlessly with Letta:
```yaml
# config.yaml with fallbacks
model_list:
- model_name: gpt-4
litellm_params:
model: openai/gpt-4
api_key: os.environ/OPENAI_API_KEY
tpm: 40000
rpm: 500
- model_name: gpt-4 # Same model name for fallback
litellm_params:
model: azure/gpt-4
api_key: os.environ/AZURE_API_KEY
api_base: os.environ/AZURE_API_BASE
api_version: "2023-07-01-preview"
tpm: 80000
rpm: 800
router_settings:
routing_strategy: "usage-based-routing"
fallbacks: [{"gpt-4": ["azure/gpt-4"]}]
```
The proxy handles all routing, load balancing, and fallbacks transparently for Letta.
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK Router">
With LiteLLM SDK, you can set up routing and fallbacks programmatically:
```python
import litellm
from litellm import Router
# Configure router with multiple models
router = Router(
model_list=[
{
"model_name": "gpt-4",
"litellm_params": {
"model": "openai/gpt-4",
"api_key": os.environ["OPENAI_API_KEY"]
},
"tpm": 40000,
"rpm": 500
},
{
"model_name": "gpt-4", # Same name for fallback
"litellm_params": {
"model": "azure/gpt-4",
"api_key": os.environ["AZURE_API_KEY"],
"api_base": os.environ["AZURE_API_BASE"],
"api_version": "2023-07-01-preview"
},
"tpm": 80000,
"rpm": 800
}
],
fallbacks=[{"gpt-4": ["azure/gpt-4"]}],
routing_strategy="usage-based-routing"
)
# Create custom completion function for Letta
def custom_completion(messages, model="gpt-4", **kwargs):
return router.completion(
model=model,
messages=messages,
**kwargs
)
# Use with Letta by monkey-patching or custom wrapper
litellm.completion = custom_completion
```
</TabItem>
</Tabs>
## Monitoring and Observability
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy Monitoring">
Enable logging to track your Letta agents' LLM usage through the proxy:
```yaml
# config.yaml with logging
model_list:
# ... your models
litellm_settings:
success_callback: ["langfuse"] # or other observability tools
environment_variables:
LANGFUSE_PUBLIC_KEY: "your-key"
LANGFUSE_SECRET_KEY: "your-secret"
```
View metrics in the proxy dashboard:
```bash
# Start proxy with UI
litellm --config config.yaml --port 4000 --detailed_debug
```
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK Monitoring">
Set up observability directly in your SDK integration:
```python
import litellm
import os
# Configure observability callbacks
os.environ["LANGFUSE_PUBLIC_KEY"] = "your-key"
os.environ["LANGFUSE_SECRET_KEY"] = "your-secret"
# Set global callbacks
litellm.success_callback = ["langfuse"]
litellm.failure_callback = ["langfuse"]
# Optional: Set up custom logging
litellm.set_verbose = True
# Create custom completion wrapper with logging
def logged_completion(messages, model="gpt-4", **kwargs):
try:
response = litellm.completion(
model=model,
messages=messages,
**kwargs
)
# Custom logging logic here if needed
return response
except Exception as e:
# Custom error handling
print(f"LLM call failed: {e}")
raise
# Use in Letta configuration
litellm.completion = logged_completion
```
</TabItem>
</Tabs>
## Example: Multi-Agent System
<Tabs>
<TabItem value="proxy" label="Using LiteLLM Proxy">
```python
import letta
from letta import create_client, LLMConfig
client = create_client()
# Create specialized agents using proxy endpoints
agents = {}
# Research agent using Claude for analysis
agents['researcher'] = client.create_agent(
name="researcher",
system="You are a research specialist. Analyze information thoroughly.",
llm_config=LLMConfig(
model="claude-3-sonnet",
model_endpoint="http://localhost:4000",
model_endpoint_type="openai"
)
)
# Writer agent using GPT-4 for content creation
agents['writer'] = client.create_agent(
name="writer",
system="You are a content writer. Create engaging, well-structured content.",
llm_config=LLMConfig(
model="gpt-4",
model_endpoint="http://localhost:4000",
model_endpoint_type="openai"
)
)
# Coordinator workflow
def research_and_write_workflow(topic: str):
# Research phase
research_response = client.user_message(
agent_id=agents['researcher'].id,
message=f"Research the topic: {topic}. Provide key insights and data."
)
research_results = research_response.messages[-1].text
# Writing phase
write_response = client.user_message(
agent_id=agents['writer'].id,
message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
)
return write_response.messages[-1].text
# Execute workflow
article = research_and_write_workflow("The future of AI in healthcare")
print(article)
```
</TabItem>
<TabItem value="sdk" label="Using LiteLLM SDK">
```python
import letta
from letta import create_client, LLMConfig
import litellm
import os
# Set up environment
os.environ["OPENAI_API_KEY"] = "your-openai-key"
os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
client = create_client()
# Create specialized agents using direct SDK models
agents = {}
# Research agent using Claude for analysis
agents['researcher'] = client.create_agent(
name="researcher",
system="You are a research specialist. Analyze information thoroughly.",
llm_config=LLMConfig(
model="anthropic/claude-3-sonnet-20240229",
model_endpoint_type="openai"
)
)
# Writer agent using GPT-4 for content creation
agents['writer'] = client.create_agent(
name="writer",
system="You are a content writer. Create engaging, well-structured content.",
llm_config=LLMConfig(
model="openai/gpt-4",
model_endpoint_type="openai"
)
)
# Cost-conscious agent using GPT-3.5
agents['reviewer'] = client.create_agent(
name="reviewer",
system="You are an editor. Review and improve content quality.",
llm_config=LLMConfig(
model="openai/gpt-3.5-turbo",
model_endpoint_type="openai"
)
)
# Enhanced workflow with multiple agents
def enhanced_workflow(topic: str):
# Research phase
research_response = client.user_message(
agent_id=agents['researcher'].id,
message=f"Research the topic: {topic}. Provide key insights and data."
)
research_results = research_response.messages[-1].text
# Writing phase
write_response = client.user_message(
agent_id=agents['writer'].id,
message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
)
draft_article = write_response.messages[-1].text
# Review phase
review_response = client.user_message(
agent_id=agents['reviewer'].id,
message=f"Please review and improve this article:\n\n{draft_article}"
)
return review_response.messages[-1].text
# Execute enhanced workflow
article = enhanced_workflow("The future of AI in healthcare")
print(article)
```
</TabItem>
</Tabs>
## Best Practices
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy Best Practices">
1. **Model Selection**: Use appropriate models for different tasks:
- Claude for analysis and reasoning
- GPT-4 for creative tasks
- GPT-3.5-turbo for simple interactions
2. **Proxy Configuration**:
- Set appropriate rate limits and timeouts
- Use fallbacks for reliability
- Enable authentication for production
3. **Memory Management**: Letta handles memory automatically, but monitor usage with large contexts
4. **Cost Optimization**:
- Use the proxy's budgeting features to control costs
- Set up rate limiting per user/team
- Monitor token usage through proxy dashboard
5. **Monitoring**: Enable observability to track agent performance and token usage
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK Best Practices">
1. **Model Selection**: Choose models based on task requirements:
- Use `openai/gpt-4` for complex reasoning
- Use `anthropic/claude-3-sonnet-20240229` for analysis
- Use `openai/gpt-3.5-turbo` for cost-effective simple tasks
2. **Error Handling**: Implement robust error handling with retries:
```python
import litellm
from litellm import completion
# Set up retry logic
litellm.num_retries = 3
litellm.request_timeout = 60
# Custom error handling
def safe_completion(**kwargs):
try:
return completion(**kwargs)
except Exception as e:
print(f"LLM call failed: {e}")
# Implement fallback logic
return completion(model="openai/gpt-3.5-turbo", **kwargs)
```
3. **Cost Management**:
- Use cheaper models for non-critical tasks
- Implement token counting and budgets
- Cache responses when appropriate
4. **Performance**:
- Use async operations for concurrent requests
- Implement connection pooling
- Monitor response times
5. **Security**:
- Store API keys securely (environment variables)
- Rotate keys regularly
- Implement rate limiting
</TabItem>
</Tabs>
## Troubleshooting
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy Issues">
### Connection Issues
```bash
# Test your LiteLLM proxy
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}]
}'
```
### Configuration Debugging
```python
# Enable verbose logging
import logging
logging.basicConfig(level=logging.DEBUG)
# Test Letta configuration
client = create_client()
print(client.get_default_llm_config())
```
### Common Proxy Issues
- **Port conflicts**: Make sure port 4000 isn't in use
- **Model not found**: Verify model names match your config.yaml
- **Authentication errors**: Check master key configuration
- **Rate limiting**: Monitor proxy logs for rate limit hits
</TabItem>
<TabItem value="sdk" label="LiteLLM SDK Issues">
### API Key Issues
```python
import os
import litellm
# Check if API keys are set
print("OpenAI Key:", os.environ.get("OPENAI_API_KEY", "Not set"))
print("Anthropic Key:", os.environ.get("ANTHROPIC_API_KEY", "Not set"))
# Test direct LiteLLM call
try:
response = litellm.completion(
model="openai/gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello"}]
)
print("LiteLLM working:", response.choices[0].message.content)
except Exception as e:
print("LiteLLM error:", e)
```
### Configuration Debugging
```python
# Enable verbose logging
litellm.set_verbose = True
# Test model availability
models = ["openai/gpt-4", "anthropic/claude-3-sonnet-20240229"]
for model in models:
try:
response = litellm.completion(
model=model,
messages=[{"role": "user", "content": "Test"}],
max_tokens=10
)
print(f"✓ {model} working")
except Exception as e:
print(f"✗ {model} failed: {e}")
```
### Common SDK Issues
- **Import errors**: Ensure `pip install litellm letta` is run
- **Model format**: Use `provider/model` format (e.g., `openai/gpt-4`)
- **API key format**: Different providers have different key formats
- **Rate limits**: Implement exponential backoff for retries
</TabItem>
</Tabs>
## Resources
- [Letta Documentation](https://docs.letta.ai/)
- [LiteLLM Proxy Documentation](../proxy/quick_start.md)
- [LiteLLM SDK Documentation](../completion/input.md)
- [Function Calling Guide](../completion/function_call.md)
- [Observability Setup](../observability/langfuse_integration.md)
- [Router Configuration](../routing.md)

2
docs/my-website/docs/mcp.md
View file

@ -114,7 +114,6 @@ mcp_servers:
description: "My custom MCP server"
auth_type: "api_key"
auth_value: "abc123"
spec_version: "2025-03-26"
```
**Configuration Options:**
@ -716,7 +715,6 @@ mcp_servers:
url: https://mcp.deepwiki.com/mcp
transport: "http"
auth_type: "none"
spec_version: "2025-03-26"
access_groups: ["dev_group"]
```

5
docs/my-website/docs/observability/helicone_integration.md
View file

@ -237,7 +237,10 @@ litellm.metadata = {
}
```
### Session Tracking and Tracing
</TabItem>
</Tabs>
## Session Tracking and Tracing
Track multi-step and agentic LLM interactions using session IDs and paths:

260
docs/my-website/docs/providers/azure_ai_img_edit.md Normal file
View file

@ -0,0 +1,260 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Azure AI Image Editing
Azure AI provides powerful image editing capabilities using FLUX models from Black Forest Labs to modify existing images based on text descriptions.
## Overview
| Property | Details |
|----------|---------|
| Description | Azure AI Image Editing uses FLUX models to modify existing images based on text prompts. |
| Provider Route on LiteLLM | `azure_ai/` |
| Provider Doc | [Azure AI FLUX Models ↗](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659) |
| Supported Operations | [`/images/edits`](#image-editing) |
## Setup
### API Key & Base URL & API Version
```python showLineNumbers
# Set your Azure AI API credentials
import os
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint" # e.g., https://your-endpoint.eastus2.inference.ai.azure.com/
os.environ["AZURE_AI_API_VERSION"] = "2025-04-01-preview" # Example API version
```
Get your API key and endpoint from [Azure AI Studio](https://ai.azure.com/).
## Supported Models
| Model Name | Description | Cost per Image |
|------------|-------------|----------------|
| `azure_ai/FLUX.1-Kontext-pro` | FLUX 1 Kontext Pro model with enhanced context understanding for editing | $0.04 |
## Image Editing
### Usage - LiteLLM Python SDK
<Tabs>
<TabItem value="basic-edit" label="Basic Usage">
```python showLineNumbers title="Basic Image Editing"
import os
import base64
from pathlib import Path
import litellm
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
os.environ["AZURE_AI_API_VERSION"] = "2025-04-01-preview"
# Edit an image with a prompt
response = litellm.image_edit(
model="azure_ai/FLUX.1-Kontext-pro",
image=open("path/to/your/image.png", "rb"),
prompt="Add a winter theme with snow and cold colors",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
api_version=os.environ["AZURE_AI_API_VERSION"]
)
img_base64 = response.data[0].get("b64_json")
img_bytes = base64.b64decode(img_base64)
path = Path("edited_image.png")
path.write_bytes(img_bytes)
```
</TabItem>
<TabItem value="async-edit" label="Async Usage">
```python showLineNumbers title="Async Image Editing"
import os
import base64
from pathlib import Path
import litellm
import asyncio
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
os.environ["AZURE_AI_API_VERSION"] = "2025-04-01-preview"
async def edit_image():
# Edit image asynchronously
response = await litellm.aimage_edit(
model="azure_ai/FLUX.1-Kontext-pro",
image=open("path/to/your/image.png", "rb"),
prompt="Make this image look like a watercolor painting",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
api_version=os.environ["AZURE_AI_API_VERSION"]
)
img_base64 = response.data[0].get("b64_json")
img_bytes = base64.b64decode(img_base64)
path = Path("async_edited_image.png")
path.write_bytes(img_bytes)
# Run the async function
asyncio.run(edit_image())
```
</TabItem>
<TabItem value="advanced-edit" label="Advanced Parameters">
```python showLineNumbers title="Advanced Image Editing with Parameters"
import os
import base64
from pathlib import Path
import litellm
# Set your API credentials
os.environ["AZURE_AI_API_KEY"] = "your-api-key-here"
os.environ["AZURE_AI_API_BASE"] = "your-azure-ai-endpoint"
os.environ["AZURE_AI_API_VERSION"] = "2025-04-01-preview"
# Edit image with additional parameters
response = litellm.image_edit(
model="azure_ai/FLUX.1-Kontext-pro",
image=open("path/to/your/image.png", "rb"),
prompt="Add magical elements like floating crystals and mystical lighting",
api_base=os.environ["AZURE_AI_API_BASE"],
api_key=os.environ["AZURE_AI_API_KEY"],
api_version=os.environ["AZURE_AI_API_VERSION"],
n=1
)
img_base64 = response.data[0].get("b64_json")
img_bytes = base64.b64decode(img_base64)
path = Path("advanced_edited_image.png")
path.write_bytes(img_bytes)
```
</TabItem>
</Tabs>
### Usage - LiteLLM Proxy Server
#### 1. Configure your config.yaml
```yaml showLineNumbers title="Azure AI Image Editing Configuration"
model_list:
- model_name: azure-flux-kontext-edit
litellm_params:
model: azure_ai/FLUX.1-Kontext-pro
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE
api_version: os.environ/AZURE_AI_API_VERSION
model_info:
mode: image_edit
general_settings:
master_key: sk-1234
```
#### 2. Start LiteLLM Proxy Server
```bash showLineNumbers title="Start LiteLLM Proxy Server"
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
#### 3. Make image editing requests with OpenAI Python SDK
<Tabs>
<TabItem value="openai-edit-sdk" label="OpenAI SDK">
```python showLineNumbers title="Azure AI Image Editing via Proxy - OpenAI SDK"
from openai import OpenAI
# Initialize client with your proxy URL
client = OpenAI(
base_url="http://localhost:4000", # Your proxy URL
api_key="sk-1234" # Your proxy API key
)
# Edit image with FLUX Kontext Pro
response = client.images.edit(
model="azure-flux-kontext-edit",
image=open("path/to/your/image.png", "rb"),
prompt="Transform this image into a beautiful oil painting style",
)
img_base64 = response.data[0].b64_json
img_bytes = base64.b64decode(img_base64)
path = Path("proxy_edited_image.png")
path.write_bytes(img_bytes)
```
</TabItem>
<TabItem value="litellm-edit-sdk" label="LiteLLM SDK">
```python showLineNumbers title="Azure AI Image Editing via Proxy - LiteLLM SDK"
import litellm
# Edit image through proxy
response = litellm.image_edit(
model="litellm_proxy/azure-flux-kontext-edit",
image=open("path/to/your/image.png", "rb"),
prompt="Add a mystical forest background with magical creatures",
api_base="http://localhost:4000",
api_key="sk-1234"
)
img_base64 = response.data[0].b64_json
img_bytes = base64.b64decode(img_base64)
path = Path("proxy_edited_image.png")
path.write_bytes(img_bytes)
```
</TabItem>
<TabItem value="curl-edit" label="cURL">
```bash showLineNumbers title="Azure AI Image Editing via Proxy - cURL"
curl --location 'http://localhost:4000/v1/images/edits' \
--header 'Authorization: Bearer sk-1234' \
--form 'model="azure-flux-kontext-edit"' \
--form 'prompt="Convert this image to a vintage sepia tone with old-fashioned effects"' \
--form 'image=@"path/to/your/image.png"'
```
</TabItem>
</Tabs>
## Supported Parameters
Azure AI Image Editing supports the following OpenAI-compatible parameters:
| Parameter | Type | Description | Default | Example |
|-----------|------|-------------|---------|---------|
| `image` | file | The image file to edit | Required | File object or binary data |
| `prompt` | string | Text description of the desired changes | Required | `"Add snow and winter elements"` |
| `model` | string | The FLUX model to use for editing | Required | `"azure_ai/FLUX.1-Kontext-pro"` |
| `n` | integer | Number of edited images to generate (You can specify only 1) | `1` | `1` |
| `api_base` | string | Your Azure AI endpoint URL | Required | `"https://your-endpoint.eastus2.inference.ai.azure.com/"` |
| `api_key` | string | Your Azure AI API key | Required | Environment variable or direct value |
| `api_version` | string | API version for Azure AI | Required | `"2025-04-01-preview"` |
## Getting Started
1. Create an account at [Azure AI Studio](https://ai.azure.com/)
2. Deploy a FLUX model in your Azure AI Studio workspace
3. Get your API key and endpoint from the deployment details
4. Set your `AZURE_AI_API_KEY`, `AZURE_AI_API_BASE` and `AZURE_AI_API_VERSION` environment variables
5. Prepare your source image
6. Use `litellm.image_edit()` to modify your images with text instructions
## Additional Resources
- [Azure AI Studio Documentation](https://docs.microsoft.com/en-us/azure/ai-services/)
- [FLUX Models Announcement](https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/black-forest-labs-flux-1-kontext-pro-and-flux1-1-pro-now-available-in-azure-ai-f/4434659)

145
docs/my-website/docs/providers/bedrock.md
View file

@ -308,6 +308,65 @@ print(response)
</TabItem>
</Tabs>
## Usage - Request Metadata
Attach metadata to Bedrock requests for logging and cost attribution.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import os
from litellm import completion
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = completion(
model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
messages=[{"role": "user", "content": "Hello, how are you?"}],
requestMetadata={
"cost_center": "engineering",
"user_id": "user123"
}
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**Set on yaml**
```yaml
model_list:
- model_name: bedrock-claude-v1
litellm_params:
model: bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0
requestMetadata:
cost_center: "engineering"
```
**Set on request**
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="bedrock-claude-v1",
messages=[{"role": "user", "content": "Hello"}],
extra_body={
"requestMetadata": {"cost_center": "engineering"}
}
)
```
</TabItem>
</Tabs>
## Usage - Function Calling / Tool calling
LiteLLM supports tool calling via Bedrock's Converse and Invoke API's.
@ -1822,6 +1881,59 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
| Mixtral 8x7B Instruct | `completion(model='bedrock/mistral.mixtral-8x7b-instruct-v0:1', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
## Bedrock Embedding
### API keys
This can be set as env variables or passed as **params to litellm.embedding()**
```python
import os
os.environ["AWS_ACCESS_KEY_ID"] = "" # Access key
os.environ["AWS_SECRET_ACCESS_KEY"] = "" # Secret access key
os.environ["AWS_REGION_NAME"] = "" # us-east-1, us-east-2, us-west-1, us-west-2
```
### Usage
```python
from litellm import embedding
response = embedding(
model="bedrock/amazon.titan-embed-text-v1",
input=["good morning from litellm"],
)
print(response)
```
#### Titan V2 - encoding_format support
```python
from litellm import embedding
# Float format (default)
response = embedding(
model="bedrock/amazon.titan-embed-text-v2:0",
input=["good morning from litellm"],
encoding_format="float" # Returns float array
)
# Binary format
response = embedding(
model="bedrock/amazon.titan-embed-text-v2:0",
input=["good morning from litellm"],
encoding_format="base64" # Returns base64 encoded binary
)
```
## Supported AWS Bedrock Embedding Models
| Model Name | Usage | Supported Additional OpenAI params |
|----------------------|---------------------------------------------|-----|
| Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | `dimensions`, `encoding_format` |
| Titan Embeddings - V1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py#L53)
| Titan Multimodal Embeddings | `embedding(model="bedrock/amazon.titan-embed-image-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py#L28) |
| Cohere Embeddings - English | `embedding(model="bedrock/cohere.embed-english-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
| Cohere Embeddings - Multilingual | `embedding(model="bedrock/cohere.embed-multilingual-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
### Advanced - [Drop Unsupported Params](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)
### Advanced - [Pass model/provider-specific Params](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)
## Image Generation
Use this for stable diffusion, and amazon nova canvas on bedrock
@ -1901,6 +2013,39 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
</TabItem>
</Tabs>
### Using Inference Profiles with Image Generation
For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import image_generation
response = image_generation(
model="bedrock/amazon.nova-canvas-v1:0",
model_id="arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
prompt="A cute baby sea otter"
)
print(f"response: {response}")
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: nova-canvas-inference-profile
litellm_params:
model: bedrock/amazon.nova-canvas-v1:0
model_id: arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0
aws_region_name: "eu-west-1"
```
</TabItem>
</Tabs>
## Supported AWS Bedrock Image Generation Models
| Model Name | Function Call |

2
docs/my-website/docs/providers/bedrock_embedding.md
View file

@ -1,4 +1,4 @@
## Bedrock Embedding
# Bedrock Embedding
## Supported Embedding Models

267
docs/my-website/docs/providers/vertex.md
View file

@ -45,7 +45,7 @@ vertex_credentials_json = json.dumps(vertex_credentials)
## COMPLETION CALL
response = completion(
model="vertex_ai/gemini-pro",
model="vertex_ai/gemini-2.5-pro",
messages=[{ "content": "Hello, how are you?","role": "user"}],
vertex_credentials=vertex_credentials_json
)
@ -69,7 +69,7 @@ vertex_credentials_json = json.dumps(vertex_credentials)
response = completion(
model="vertex_ai/gemini-pro",
model="vertex_ai/gemini-2.5-pro",
messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}],
vertex_credentials=vertex_credentials_json
)
@ -189,7 +189,7 @@ print(json.loads(completion.choices[0].message.content))
1. Add model to config.yaml
```yaml
model_list:
- model_name: gemini-pro
- model_name: gemini-2.5-pro
litellm_params:
model: vertex_ai/gemini-1.5-pro
vertex_project: "project-id"
@ -210,7 +210,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "gemini-pro",
"model": "gemini-2.5-pro",
"messages": [
{"role": "user", "content": "List 5 popular cookie recipes."}
],
@ -262,7 +262,7 @@ except JSONSchemaValidationError as e:
1. Add model to config.yaml
```yaml
model_list:
- model_name: gemini-pro
- model_name: gemini-2.5-pro
litellm_params:
model: vertex_ai/gemini-1.5-pro
vertex_project: "project-id"
@ -283,7 +283,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "gemini-pro",
"model": "gemini-2.5-pro",
"messages": [
{"role": "user", "content": "List 5 popular cookie recipes."}
],
@ -391,7 +391,7 @@ client = OpenAI(
)
response = client.chat.completions.create(
model="gemini-pro",
model="gemini-2.5-pro",
messages=[{"role": "user", "content": "Who won the world cup?"}],
tools=[{"googleSearch": {}}],
)
@ -406,7 +406,7 @@ curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-pro",
"model": "gemini-2.5-pro",
"messages": [
{"role": "user", "content": "Who won the world cup?"}
],
@ -527,7 +527,7 @@ client = OpenAI(
)
response = client.chat.completions.create(
model="gemini-pro",
model="gemini-2.5-pro",
messages=[{"role": "user", "content": "Who won the world cup?"}],
tools=[{"enterpriseWebSearch": {}}],
)
@ -542,7 +542,7 @@ curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"model": "gemini-pro",
"model": "gemini-2.5-pro",
"messages": [
{"role": "user", "content": "Who won the world cup?"}
],
@ -815,6 +815,77 @@ Use Vertex AI context caching is supported by calling provider api directly. (Un
[**Go straight to provider**](../pass_through/vertex_ai.md#context-caching)
#### 1. Create the Cache
First, create the cache by sending a `POST` request to the `cachedContents` endpoint via the LiteLLM proxy.
<Tabs>
<TabItem value="proxy" label="PROXY">
```bash
curl http://0.0.0.0:4000/vertex_ai/v1/projects/{project_id}/locations/{location}/cachedContents \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "projects/{project_id}/locations/{location}/publishers/google/models/gemini-2.5-flash",
"displayName": "example_cache",
"contents": [{
"role": "user",
"parts": [{
"text": ".... a long book to be cached"
}]
}]
}'
```
</TabItem>
</Tabs>
#### 2. Get the Cache Name from the Response
Vertex AI will return a response containing the `name` of the cached content. This name is the identifier for your cached data.
```json
{
"name": "projects/12341234/locations/{location}/cachedContents/123123123123123",
"model": "projects/{project_id}/locations/{location}/publishers/google/models/gemini-2.5-flash",
"createTime": "2025-09-23T19:13:50.674976Z",
"updateTime": "2025-09-23T19:13:50.674976Z",
"expireTime": "2025-09-23T20:13:50.655988Z",
"displayName": "example_cache",
"usageMetadata": {
"totalTokenCount": 1246,
"textCount": 5132
}
}
```
#### 3. Use the Cached Content
Use the `name` from the response as `cachedContent` or `cached_content` in subsequent API calls to reuse the cached information. This is passed in the body of your request to `/chat/completions`.
<Tabs>
<TabItem value="proxy" label="PROXY">
```bash
curl http://0.0.0.0:4000/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"cachedContent": "projects/545201925769/locations/us-central1/cachedContents/4511135542628319232",
"model": "gemini-2.5-flash",
"messages": [
{
"role": "user",
"content": "what is the book about?"
}
]
}'
```
</TabItem>
## Pre-requisites
* `pip install google-cloud-aiplatform` (pre-installed on proxy docker image)
@ -835,7 +906,7 @@ import litellm
litellm.vertex_project = "hardy-device-38811" # Your Project ID
litellm.vertex_location = "us-central1" # proj location
response = litellm.completion(model="gemini-pro", messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}])
response = litellm.completion(model="gemini-2.5-pro", messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}])
```
## Usage with LiteLLM Proxy Server
@ -876,9 +947,9 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server
vertex_location: "us-central1" # proj location
model_list:
-model_name: team1-gemini-pro
-model_name: team1-gemini-2.5-pro
litellm_params:
model: gemini-pro
model: gemini-2.5-pro
```
</TabItem>
@ -905,7 +976,7 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server
)
response = client.chat.completions.create(
model="team1-gemini-pro",
model="team1-gemini-2.5-pro",
messages = [
{
"role": "user",
@ -925,7 +996,7 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "team1-gemini-pro",
"model": "team1-gemini-2.5-pro",
"messages": [
{
"role": "user",
@ -975,7 +1046,7 @@ vertex_credentials_json = json.dumps(vertex_credentials)
response = completion(
model="vertex_ai/gemini-pro",
model="vertex_ai/gemini-2.5-pro",
messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}],
vertex_credentials=vertex_credentials_json,
vertex_project="my-special-project",
@ -1039,7 +1110,7 @@ In certain use-cases you may need to make calls to the models and pass [safety s
```python
response = completion(
model="vertex_ai/gemini-pro",
model="vertex_ai/gemini-2.5-pro",
messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}]
safety_settings=[
{
@ -1153,7 +1224,7 @@ litellm.vertex_ai_safety_settings = [
},
]
response = completion(
model="vertex_ai/gemini-pro",
model="vertex_ai/gemini-2.5-pro",
messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}]
)
```
@ -1212,7 +1283,7 @@ litellm.vertex_location = "us-central1 # Your Location
## Gemini Pro
| Model Name | Function Call |
|------------------|--------------------------------------|
| gemini-pro | `completion('gemini-pro', messages)`, `completion('vertex_ai/gemini-pro', messages)` |
| gemini-2.5-pro | `completion('gemini-2.5-pro', messages)`, `completion('vertex_ai/gemini-2.5-pro', messages)` |
## Fine-tuned Models
@ -1307,7 +1378,7 @@ curl --location 'https://0.0.0.0:4000/v1/chat/completions' \
## Gemini Pro Vision
| Model Name | Function Call |
|------------------|--------------------------------------|
| gemini-pro-vision | `completion('gemini-pro-vision', messages)`, `completion('vertex_ai/gemini-pro-vision', messages)`|
| gemini-2.5-pro-vision | `completion('gemini-2.5-pro-vision', messages)`, `completion('vertex_ai/gemini-2.5-pro-vision', messages)`|
## Gemini 1.5 Pro (and Vision)
| Model Name | Function Call |
@ -1321,7 +1392,7 @@ curl --location 'https://0.0.0.0:4000/v1/chat/completions' \
#### Using Gemini Pro Vision
Call `gemini-pro-vision` in the same input/output format as OpenAI [`gpt-4-vision`](https://docs.litellm.ai/docs/providers/openai#openai-vision-models)
Call `gemini-2.5-pro-vision` in the same input/output format as OpenAI [`gpt-4-vision`](https://docs.litellm.ai/docs/providers/openai#openai-vision-models)
LiteLLM Supports the following image types passed in `url`
- Images with Cloud Storage URIs - gs://cloud-samples-data/generative-ai/image/boats.jpeg
@ -1339,7 +1410,7 @@ LiteLLM Supports the following image types passed in `url`
import litellm
response = litellm.completion(
model = "vertex_ai/gemini-pro-vision",
model = "vertex_ai/gemini-2.5-pro-vision",
messages=[
{
"role": "user",
@ -1377,7 +1448,7 @@ image_path = "cached_logo.jpg"
# Getting the base64 string
base64_image = encode_image(image_path)
response = litellm.completion(
model="vertex_ai/gemini-pro-vision",
model="vertex_ai/gemini-2.5-pro-vision",
messages=[
{
"role": "user",
@ -1433,7 +1504,7 @@ tools = [
messages = [{"role": "user", "content": "What's the weather like in Boston today?"}]
response = completion(
model="vertex_ai/gemini-pro-vision",
model="vertex_ai/gemini-2.5-pro-vision",
messages=messages,
tools=tools,
)
@ -2509,150 +2580,6 @@ print("response from proxy", response)
</TabItem>
</Tabs>
## **Batch APIs**
Just add the following Vertex env vars to your environment.
```bash
# GCS Bucket settings, used to store batch prediction files in
export GCS_BUCKET_NAME = "litellm-testing-bucket" # the bucket you want to store batch prediction files in
export GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json" # path to your service account json file
# Vertex /batch endpoint settings, used for LLM API requests
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" # path to your service account json file
export VERTEXAI_LOCATION="us-central1" # can be any vertex location
export VERTEXAI_PROJECT="my-test-project"
```
### Usage
#### 1. Create a file of batch requests for vertex
LiteLLM expects the file to follow the **[OpenAI batches files format](https://platform.openai.com/docs/guides/batch)**
Each `body` in the file should be an **OpenAI API request**
Create a file called `vertex_batch_completions.jsonl` in the current working directory, the `model` should be the Vertex AI model name
```
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
```
#### 2. Upload a File of batch requests
For `vertex_ai` litellm will upload the file to the provided `GCS_BUCKET_NAME`
```python
import os
oai_client = OpenAI(
api_key="sk-1234", # litellm proxy API key
base_url="http://localhost:4000" # litellm proxy base url
)
file_name = "vertex_batch_completions.jsonl" #
_current_dir = os.path.dirname(os.path.abspath(__file__))
file_path = os.path.join(_current_dir, file_name)
file_obj = oai_client.files.create(
file=open(file_path, "rb"),
purpose="batch",
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use vertex_ai for this file upload
)
```
**Expected Response**
```json
{
"id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
"bytes": 416,
"created_at": 1733392026,
"filename": "litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
"object": "file",
"purpose": "batch",
"status": "uploaded",
"status_details": null
}
```
#### 3. Create a batch
```python
batch_input_file_id = file_obj.id # use `file_obj` from step 2
create_batch_response = oai_client.batches.create(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id=batch_input_file_id, # example input_file_id = "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/c2b1b785-252b-448c-b180-033c4c63b3ce"
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use `vertex_ai` for this batch request
)
```
**Expected Response**
```json
{
"id": "3814889423749775360",
"completion_window": "24hrs",
"created_at": 1733392026,
"endpoint": "",
"input_file_id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
"object": "batch",
"status": "validating",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001",
"request_counts": null
}
```
#### 4. Retrieve a batch
```python
retrieved_batch = oai_client.batches.retrieve(
batch_id=create_batch_response.id,
extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use `vertex_ai` for this batch request
)
```
**Expected Response**
```json
{
"id": "3814889423749775360",
"completion_window": "24hrs",
"created_at": 1736500100,
"endpoint": "",
"input_file_id": "gs://example-bucket-1-litellm/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/7b2e47f5-3dd4-436d-920f-f9155bbdc952",
"object": "batch",
"status": "completed",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://example-bucket-1-litellm/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001",
"request_counts": null
}
```
## **Fine Tuning APIs**
@ -2868,7 +2795,3 @@ Once that's done, when you deploy the new container in the Google Cloud Run serv
s/o @[Darien Kindlund](https://www.linkedin.com/in/kindlund/) for this tutorial

264
docs/my-website/docs/providers/vertex_batch.md Normal file
View file

@ -0,0 +1,264 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## **Batch APIs**
Just add the following Vertex env vars to your environment.
```bash
# GCS Bucket settings, used to store batch prediction files in
export GCS_BUCKET_NAME="my-batch-bucket" # the bucket you want to store batch prediction files in
export GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json" # path to your service account json file
# Vertex /batch endpoint settings, used for LLM API requests
export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" # path to your service account json file
export VERTEXAI_LOCATION="us-central1" # can be any vertex location
export VERTEXAI_PROJECT="my-project"
```
### Usage
Follow this complete workflow: create JSONL file → upload file → create batch → retrieve batch status → get file content
#### 1. Create a JSONL file of batch requests
LiteLLM expects the file to follow the **[OpenAI batches files format](https://platform.openai.com/docs/guides/batch)**.
Each `body` in the file should be an **OpenAI API request**.
Create a file called `batch_requests.jsonl` with your requests:
```jsonl
{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-2.5-flash-lite", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-2.5-flash-lite", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
```
#### 2. Upload the file
Upload your JSONL file. For `vertex_ai`, the file will be stored in your configured GCS bucket provided by `GCS_BUCKET_NAME`.
<Tabs>
<TabItem value="python" label="Python">
```python showLineNumbers title="upload_file.py"
from openai import OpenAI
oai_client = OpenAI(
api_key="sk-1234", # litellm proxy API key
base_url="http://localhost:4000" # litellm proxy base url
)
file_obj = oai_client.files.create(
file=open("batch_requests.jsonl", "rb"),
purpose="batch",
extra_body={"custom_llm_provider": "vertex_ai"}
)
print(f"File uploaded with ID: {file_obj.id}")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Upload File"
curl --request POST \
--url http://localhost:4000/v1/files \
--header 'Content-Type: multipart/form-data' \
--form purpose=batch \
--form file=@batch_requests.jsonl \
--form custom_llm_provider=vertex_ai
```
</TabItem>
</Tabs>
**Expected Response:**
```json
{
"id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"bytes": 416,
"created_at": 1758303684,
"filename": "litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"object": "file",
"purpose": "batch",
"status": "uploaded",
"expires_at": null,
"status_details": null
}
```
#### 3. Create a batch
Create a batch job using the uploaded file ID.
<Tabs>
<TabItem value="python" label="Python">
```python showLineNumbers title="create_batch.py"
batch_input_file_id = file_obj.id # from step 2
create_batch_response = oai_client.batches.create(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id=batch_input_file_id, # e.g. "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd"
extra_body={"custom_llm_provider": "vertex_ai"}
)
print(f"Batch created with ID: {create_batch_response.id}")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Create Batch Request"
curl --request POST \
--url http://localhost:4000/v1/batches \
--header 'Content-Type: application/json' \
--data '{
"input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"endpoint": "/v1/chat/completions",
"completion_window": "24h",
"custom_llm_provider": "vertex_ai"
}'
```
</TabItem>
</Tabs>
**Expected Response:**
```json
{
"id": "7814463557919047680",
"completion_window": "24hrs",
"created_at": 1758328011,
"endpoint": "",
"input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"object": "batch",
"status": "validating",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite",
"request_counts": null,
"usage": null
}
```
#### 4. Retrieve batch status
Check the status of your batch job. The batch will progress through states: `validating` → `in_progress` → `completed`.
<Tabs>
<TabItem value="python" label="Python">
```python showLineNumbers title="retrieve_batch.py"
retrieved_batch = oai_client.batches.retrieve(
batch_id=create_batch_response.id, # Created batch id, e.g. 7814463557919047680
extra_body={"custom_llm_provider": "vertex_ai"}
)
print(f"Batch status: {retrieved_batch.status}")
if retrieved_batch.status == "completed":
print(f"Output file: {retrieved_batch.output_file_id}")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Retrieve Batch Status"
curl --request GET \
--url 'http://localhost:4000/batches/7814463557919047680?provider=vertex_ai' \
--header 'Authorization: Bearer sk-1234'
```
</TabItem>
</Tabs>
**Expected Response (when completed):**
```json
{
"id": "7814463557919047680",
"completion_window": "24hrs",
"created_at": 1758328011,
"endpoint": "",
"input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
"object": "batch",
"status": "completed",
"cancelled_at": null,
"cancelling_at": null,
"completed_at": null,
"error_file_id": null,
"errors": null,
"expired_at": null,
"expires_at": null,
"failed_at": null,
"finalizing_at": null,
"in_progress_at": null,
"metadata": null,
"output_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/prediction-model-2025-09-19T21:26:51.569037Z/predictions.jsonl",
"request_counts": null,
"usage": null
}
```
#### 5. Get file content
Once the batch is completed, retrieve the results using the `output_file_id` from the batch response.
**Important:** The `output_file_id` must be URL encoded when used in the request path.
<Tabs>
<TabItem value="python" label="Python">
```python showLineNumbers title="get_file_content.py"
import urllib.parse
import json
output_file_id = retrieved_batch.output_file_id
# URL encode the file ID
encoded_file_id = urllib.parse.quote_plus(output_file_id)
# Get file content
file_content = oai_client.files.content(
file_id=encoded_file_id,
extra_body={"custom_llm_provider": "vertex_ai"}
)
# Process the results
for line in file_content.text.strip().split('\n'):
result = json.loads(line)
print(f"Request: {result['request']}")
print(f"Response: {result['response']}")
print("---")
```
</TabItem>
<TabItem value="curl" label="Curl">
```bash showLineNumbers title="Get File Content"
# Note: The file ID must be URL encoded
curl --request GET \
--url 'http://localhost:4000/files/gs%253A%252F%252Fmy-batch-bucket%252Flitellm-vertex-files%252Fpublishers%252Fgoogle%252Fmodels%252Fgemini-2.5-flash-lite%252Fprediction-model-2025-09-19T21%253A26%253A51.569037Z%252Fpredictions.jsonl/content?provider=vertex_ai' \
--header 'Authorization: Bearer sk-1234'
```
</TabItem>
</Tabs>
**Expected Response:**
The response contains JSONL format with one result per line:
```jsonl
{"status":"","processed_time":"2025-09-19T21:29:47.352+00:00","request":{"contents":[{"parts":[{"text":"Hello world!"}],"role":"user"}],"generationConfig":{"max_output_tokens":10},"system_instruction":{"parts":[{"text":"You are a helpful assistant."}]}},"response":{"candidates":[{"avgLogprobs":-0.48079710006713866,"content":{"parts":[{"text":"Hello there! It's nice to meet you"}],"role":"model"},"finishReason":"MAX_TOKENS"}],"createTime":"2025-09-19T21:29:47.484619Z","modelVersion":"gemini-2.5-flash-lite","responseId":"S8vNaIvKHdvshMIP_aOtuAg","usageMetadata":{"candidatesTokenCount":10,"candidatesTokensDetails":[{"modality":"TEXT","tokenCount":10}],"promptTokenCount":9,"promptTokensDetails":[{"modality":"TEXT","tokenCount":9}],"totalTokenCount":19,"trafficType":"ON_DEMAND"}}}
{"status":"","processed_time":"2025-09-19T21:29:47.358+00:00","request":{"contents":[{"parts":[{"text":"Hello world!"}],"role":"user"}],"generationConfig":{"max_output_tokens":10},"system_instruction":{"parts":[{"text":"You are an unhelpful assistant."}]}},"response":{"candidates":[{"avgLogprobs":-0.6168075137668185,"content":{"parts":[{"text":"I am unable to assist with this request."}],"role":"model"},"finishReason":"STOP"}],"createTime":"2025-09-19T21:29:47.470889Z","modelVersion":"gemini-2.5-flash-lite","responseId":"S8vNaOneHISShMIP28nA8QQ","usageMetadata":{"candidatesTokenCount":9,"candidatesTokensDetails":[{"modality":"TEXT","tokenCount":9}],"promptTokenCount":9,"promptTokensDetails":[{"modality":"TEXT","tokenCount":9}],"totalTokenCount":18,"trafficType":"ON_DEMAND"}}}
```

1
docs/my-website/docs/proxy/budget_reset_and_tz.md
View file

@ -29,5 +29,6 @@ Common timezone values:
- `US/Pacific` - Pacific Time
- `Europe/London` - UK Time
- `Asia/Kolkata` - Indian Standard Time (IST)
- `Asia/Bangkok` - Indochina Time (ICT)
- `Asia/Tokyo` - Japan Standard Time
- `Australia/Sydney` - Australian Eastern Time

163
docs/my-website/docs/proxy/custom_auth.md
View file

@ -21,6 +21,169 @@ async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
raise Exception
```
## UserAPIKeyAuth Fields Reference
The `UserAPIKeyAuth` object supports the following fields for comprehensive auth configuration:
### Core Authentication Fields
```python
UserAPIKeyAuth(
# Basic auth fields
api_key: Optional[str] = None, # The API key (will be hashed automatically)
token: Optional[str] = None, # Hashed token for internal use
key_name: Optional[str] = None, # Human-readable key name
key_alias: Optional[str] = None, # Key alias for identification
# User identification
user_id: Optional[str] = None, # Unique user identifier
user_email: Optional[str] = None, # User email address
user_role: Optional[LitellmUserRoles] = None, # User role (PROXY_ADMIN, INTERNAL_USER, etc.)
# Team/Organization
team_id: Optional[str] = None, # Team identifier
team_alias: Optional[str] = None, # Team display name
org_id: Optional[str] = None, # Organization identifier
)
```
### Budget and Spend Tracking
```python
UserAPIKeyAuth(
# User budgets
max_budget: Optional[float] = None, # Maximum budget for the key
spend: float = 0.0, # Current spend amount
soft_budget: Optional[float] = None, # Soft budget limit (warnings)
model_max_budget: Dict = {}, # Per-model budget limits
model_spend: Dict = {}, # Per-model spend tracking
# Team budgets
team_max_budget: Optional[float] = None, # Team's maximum budget
team_spend: Optional[float] = None, # Team's current spend
team_member_spend: Optional[float] = None, # This user's spend within the team
# Budget timing
budget_duration: Optional[str] = None, # Budget reset period
budget_reset_at: Optional[datetime] = None, # When budget resets
)
```
### Rate Limiting
```python
UserAPIKeyAuth(
# User limits
tpm_limit: Optional[int] = None, # Tokens per minute limit
rpm_limit: Optional[int] = None, # Requests per minute limit
user_tpm_limit: Optional[int] = None, # User-specific TPM limit
user_rpm_limit: Optional[int] = None, # User-specific RPM limit
# Team limits
team_tpm_limit: Optional[int] = None, # Team TPM limit
team_rpm_limit: Optional[int] = None, # Team RPM limit
team_member_tpm_limit: Optional[int] = None, # Per-member TPM limit
team_member_rpm_limit: Optional[int] = None, # Per-member RPM limit
# Per-model limits
rpm_limit_per_model: Optional[Dict[str, int]] = None, # RPM limits by model
tpm_limit_per_model: Optional[Dict[str, int]] = None, # TPM limits by model
)
```
### End User Tracking
```python
UserAPIKeyAuth(
# End user identification and limits
end_user_id: Optional[str] = None, # End user identifier
end_user_tpm_limit: Optional[int] = None, # End user TPM limit
end_user_rpm_limit: Optional[int] = None, # End user RPM limit
end_user_max_budget: Optional[float] = None, # End user budget limit
)
```
### Model and Route Access
```python
UserAPIKeyAuth(
# Model access control
models: List = [], # Allowed models list
team_models: List = [], # Team's allowed models
aliases: Dict = {}, # Model aliases
# Route permissions
allowed_routes: Optional[list] = [], # Allowed API routes
allowed_cache_controls: Optional[list] = [], # Cache control permissions
permissions: Dict = {}, # General permissions
)
```
### Advanced Configuration
```python
UserAPIKeyAuth(
# Request handling
max_parallel_requests: Optional[int] = None, # Concurrent request limit
allowed_model_region: Optional[AllowedModelRegion] = None, # Geographic restrictions
# Expiration and status
expires: Optional[Union[str, datetime]] = None, # Key expiration
blocked: Optional[bool] = None, # Whether key is blocked
# Metadata and configuration
metadata: Dict = {}, # Custom metadata
config: Dict = {}, # Configuration settings
team_metadata: Optional[Dict] = None, # Team metadata
# Internal tracking
request_route: Optional[str] = None, # Current request route
last_refreshed_at: Optional[float] = None, # Cache refresh timestamp
)
```
### Complete Example
```python
from datetime import datetime, timedelta
from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles
async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
try:
# Example: Comprehensive auth configuration
if api_key.startswith("sk-admin-"):
return UserAPIKeyAuth(
api_key=api_key,
user_id="admin_user_123",
user_email="admin@company.com",
user_role=LitellmUserRoles.PROXY_ADMIN,
team_id="admin_team",
team_alias="Administrative Team",
max_budget=1000.0,
soft_budget=800.0,
tpm_limit=10000,
rpm_limit=100,
models=["gpt-4", "claude-3-sonnet", "gpt-3.5-turbo"],
allowed_routes=["/chat/completions", "/embeddings"],
expires=datetime.now() + timedelta(days=30),
metadata={"department": "engineering", "cost_center": "ai_ops"}
)
elif api_key.startswith("sk-team-"):
return UserAPIKeyAuth(
api_key=api_key,
user_id="team_user_456",
user_email="user@company.com",
user_role=LitellmUserRoles.INTERNAL_USER,
team_id="dev_team",
team_alias="Development Team",
max_budget=100.0,
tpm_limit=1000,
rpm_limit=20,
models=["gpt-3.5-turbo", "claude-3-haiku"],
team_member_tpm_limit=500, # Limit within team
end_user_tpm_limit=100, # Per end-user limit
metadata={"project": "chatbot_v2"}
)
else:
raise Exception("Invalid API key")
except Exception:
raise Exception("Authentication failed")
```
#### 2. Pass the filepath (relative to the config.yaml)
Pass the filepath to the config.yaml

1
docs/my-website/docs/proxy/deploy.md
View file

@ -13,6 +13,7 @@ To start using Litellm, run the following commands in a shell:
```bash
# Get the code
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/prometheus.yml
# Add the master key - you can change this after setup
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env

255
docs/my-website/docs/proxy/dynamic_rate_limit.md Normal file
View file

@ -0,0 +1,255 @@
# Dynamic TPM/RPM Allocation
Prevent projects from gobbling too much tpm/rpm.
Dynamically allocate TPM/RPM quota to api keys, based on active keys in that minute. [**See Code**](https://github.com/BerriAI/litellm/blob/9bffa9a48e610cc6886fc2dce5c1815aeae2ad46/litellm/proxy/hooks/dynamic_rate_limiter.py#L125)
## Quick Start Usage
1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: my-fake-model
litellm_params:
model: gpt-3.5-turbo
api_key: my-fake-key
mock_response: hello-world
tpm: 60
litellm_settings:
callbacks: ["dynamic_rate_limiter_v3"]
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
database_url: postgres://.. # OR set `DATABASE_URL=".."` in your .env
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```python showLineNumbers title="test.py"
"""
- Run 2 concurrent teams calling same model
- model has 60 TPM
- Mock response returns 30 total tokens / request
- Each team will only be able to make 1 request per minute
"""
import requests
from openai import OpenAI, RateLimitError
def create_key(api_key: str, base_url: str):
response = requests.post(
url="{}/key/generate".format(base_url),
json={},
headers={
"Authorization": "Bearer {}".format(api_key)
}
)
_response = response.json()
return _response["key"]
key_1 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
key_2 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# call proxy with key 1 - works
openai_client_1 = OpenAI(api_key=key_1, base_url="http://0.0.0.0:4000")
response = openai_client_1.chat.completions.with_raw_response.create(
model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
)
print("Headers for call 1 - {}".format(response.headers))
_response = response.parse()
print("Total tokens for call - {}".format(_response.usage.total_tokens))
# call proxy with key 2 - works
openai_client_2 = OpenAI(api_key=key_2, base_url="http://0.0.0.0:4000")
response = openai_client_2.chat.completions.with_raw_response.create(
model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
)
print("Headers for call 2 - {}".format(response.headers))
_response = response.parse()
print("Total tokens for call - {}".format(_response.usage.total_tokens))
# call proxy with key 2 - fails
try:
openai_client_2.chat.completions.with_raw_response.create(model="my-fake-model", messages=[{"role": "user", "content": "Hey, how's it going?"}])
raise Exception("This should have failed!")
except RateLimitError as e:
print("This was rate limited b/c - {}".format(str(e)))
```
**Expected Response**
```
This was rate limited b/c - Error code: 429 - {'error': {'message': {'error': 'Key=<hashed_token> over available TPM=0. Model TPM=0, Active keys=2'}, 'type': 'None', 'param': 'None', 'code': 429}}
```
## [BETA] Set Priority / Reserve Quota
Reserve TPM/RPM capacity for different environments or use cases. This ensures critical production workloads always have guaranteed capacity, while development or lower-priority tasks use remaining quota.
**Use Cases:**
- Production vs Development environments
- Real-time applications vs batch processing
- Critical services vs experimental features
:::tip
Reserving TPM/RPM on keys based on priority is a premium feature. Please [get an enterprise license](./enterprise.md) for it.
:::
### How Priority Reservation Works
Priority reservation allocates a percentage of your model's total TPM/RPM to specific priority levels. Keys with higher priority get guaranteed access to their reserved quota first.
**Example Scenario:**
- Model has 10 RPM total capacity
- Priority reservation: `{"prod": 0.9, "dev": 0.1}`
- Result: Production keys get 9 RPM guaranteed, Development keys get 1 RPM guaranteed
### Configuration
#### 1. Setup config.yaml
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: "gpt-3.5-turbo"
api_key: os.environ/OPENAI_API_KEY
rpm: 10 # Total model capacity
litellm_settings:
callbacks: ["dynamic_rate_limiter_v3"]
priority_reservation:
"prod": 0.9 # 90% reserved for production (9 RPM)
"dev": 0.1 # 10% reserved for development (1 RPM)
priority_reservation_settings:
default_priority: 0 # Weight (0%) assigned to keys without explicit priority metadata
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
database_url: postgres://.. # OR set `DATABASE_URL=".."` in your.env
```
**Configuration Details:**
`priority_reservation`: Dict[str, float]
- **Key (str)**: Priority level name (can be any string like "prod", "dev", "critical", etc.)
- **Value (float)**: Percentage of total TPM/RPM to reserve (0.0 to 1.0)
- **Note**: Values should sum to 1.0 or less
`priority_reservation_settings`: Object (Optional)
- **default_priority (float)**: Weight/percentage (0.0 to 1.0) assigned to API keys that have no priority metadata set (defaults to 0.5)
**Start Proxy**
```bash
litellm --config /path/to/config.yaml
```
#### 2. Create Keys with Priority Levels
**Production Key:**
```bash
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {"priority": "prod"}
}'
```
**Development Key:**
```bash
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{
"metadata": {"priority": "dev"}
}'
```
**Key Without Priority (uses default_priority weight):**
```bash
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-d '{}'
```
**Expected Response for both:**
```json
{
"key": "sk-...",
"metadata": {"priority": "prod"}, // or "dev"
...
}
```
#### 3. Test Priority Allocation
**Test Production Key (should get 9 RPM):**
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-prod-key' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello from prod"}]
}'
```
**Test Development Key (should get 1 RPM):**
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-dev-key' \
-d '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello from dev"}]
}'
```
### Expected Behavior
With the configuration above:
1. **Production keys** can make up to 9 requests per minute (90% of 10 RPM)
2. **Development keys** can make up to 1 request per minute (10% of 10 RPM)
3. **Keys without explicit priority** get the default_priority weight (0 = 0%), which allocates 0 requests per minute (0% of 10 RPM)
4. Named priorities in `priority_reservation` and keys with `default_priority` operate independently
**Rate Limit Error Example:**
```json
{
"error": {
"message": "Key=sk-dev-... over available RPM=0. Model RPM=10, Reserved RPM for priority 'dev'=1, Active keys=1",
"type": "rate_limit_exceeded",
"code": 429
}
}
```
### Demo Video
This video walks through setting up dynamic rate limiting with priority reservation and locust tests to validate the behavior.
<iframe width="840" height="500" src="https://www.loom.com/embed/1b54b93139ee415d959402cc0629f3f7
" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>

185
docs/my-website/docs/proxy/team_budgets.md
View file

@ -178,188 +178,3 @@ Expect to see this metric on prometheus to track the Remaining Budget for the te
```shell
litellm_remaining_team_budget_metric{team_alias="QA Prod Bot",team_id="de35b29e-6ca8-4f47-b804-2b79d07aa99a"} 9.699999999999992e-06
```
### Dynamic TPM/RPM Allocation
Prevent projects from gobbling too much tpm/rpm.
Dynamically allocate TPM/RPM quota to api keys, based on active keys in that minute. [**See Code**](https://github.com/BerriAI/litellm/blob/9bffa9a48e610cc6886fc2dce5c1815aeae2ad46/litellm/proxy/hooks/dynamic_rate_limiter.py#L125)
1. Setup config.yaml
```yaml
model_list:
- model_name: my-fake-model
litellm_params:
model: gpt-3.5-turbo
api_key: my-fake-key
mock_response: hello-world
tpm: 60
litellm_settings:
callbacks: ["dynamic_rate_limiter"]
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
database_url: postgres://.. # OR set `DATABASE_URL=".."` in your .env
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```python
"""
- Run 2 concurrent teams calling same model
- model has 60 TPM
- Mock response returns 30 total tokens / request
- Each team will only be able to make 1 request per minute
"""
import requests
from openai import OpenAI, RateLimitError
def create_key(api_key: str, base_url: str):
response = requests.post(
url="{}/key/generate".format(base_url),
json={},
headers={
"Authorization": "Bearer {}".format(api_key)
}
)
_response = response.json()
return _response["key"]
key_1 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
key_2 = create_key(api_key="sk-1234", base_url="http://0.0.0.0:4000")
# call proxy with key 1 - works
openai_client_1 = OpenAI(api_key=key_1, base_url="http://0.0.0.0:4000")
response = openai_client_1.chat.completions.with_raw_response.create(
model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
)
print("Headers for call 1 - {}".format(response.headers))
_response = response.parse()
print("Total tokens for call - {}".format(_response.usage.total_tokens))
# call proxy with key 2 - works
openai_client_2 = OpenAI(api_key=key_2, base_url="http://0.0.0.0:4000")
response = openai_client_2.chat.completions.with_raw_response.create(
model="my-fake-model", messages=[{"role": "user", "content": "Hello world!"}],
)
print("Headers for call 2 - {}".format(response.headers))
_response = response.parse()
print("Total tokens for call - {}".format(_response.usage.total_tokens))
# call proxy with key 2 - fails
try:
openai_client_2.chat.completions.with_raw_response.create(model="my-fake-model", messages=[{"role": "user", "content": "Hey, how's it going?"}])
raise Exception("This should have failed!")
except RateLimitError as e:
print("This was rate limited b/c - {}".format(str(e)))
```
**Expected Response**
```
This was rate limited b/c - Error code: 429 - {'error': {'message': {'error': 'Key=<hashed_token> over available TPM=0. Model TPM=0, Active keys=2'}, 'type': 'None', 'param': 'None', 'code': 429}}
```
#### ✨ [BETA] Set Priority / Reserve Quota
Reserve tpm/rpm capacity for projects in prod.
:::tip
Reserving tpm/rpm on keys based on priority is a premium feature. Please [get an enterprise license](./enterprise.md) for it.
:::
1. Setup config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: "gpt-3.5-turbo"
api_key: os.environ/OPENAI_API_KEY
rpm: 100
litellm_settings:
callbacks: ["dynamic_rate_limiter"]
priority_reservation: {"dev": 0, "prod": 1}
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
database_url: postgres://.. # OR set `DATABASE_URL=".."` in your .env
```
priority_reservation:
- Dict[str, float]
- str: can be any string
- float: from 0 to 1. Specify the % of tpm/rpm to reserve for keys of this priority.
**Start Proxy**
```
litellm --config /path/to/config.yaml
```
2. Create a key with that priority
```bash
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer <your-master-key>' \
-H 'Content-Type: application/json' \
-D '{
"metadata": {"priority": "dev"} # 👈 KEY CHANGE
}'
```
**Expected Response**
```
{
...
"key": "sk-.."
}
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: sk-...' \ # 👈 key from step 2.
-d '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
}'
```
**Expected Response**
```
Key=... over available RPM=0. Model RPM=100, Active keys=None
```

82
docs/my-website/docs/proxy/user_onboarding.md Normal file
View file

@ -0,0 +1,82 @@
# User Onboarding Guide
A step-by-step guide to help admins onboard users to your LiteLLM proxy instance and help users get started with their API key.
---
## For Administrators
### Step 1: Create a User Account
You can create a user account via the Admin UI or using the API.
#### Admin UI
- Go to the (`/ui` endpoint)
- Navigate to the Internal Users section
- Click "Add User" and fill in the required details
#### API
```bash
curl -X POST http://localhost:4000/user/new \
-H "Authorization: Bearer <admin-key>" \
-H "Content-Type: application/json" \
-d '{"user_email": "user@example.com"}'
```
---
### Step 2: Grant Access & Permissions
- Assign the user to a team (optional)
- Set budgets, rate limits, and allowed models as needed
- Generate an API key for the user (via UI or API)
#### **Generate API Key (API Example)**
```bash
curl -X POST http://localhost:4000/key/generate \
-H "Authorization: Bearer <admin-key>" \
-H "Content-Type: application/json" \
-d '{"user_id": "<user-id>", "max_budget": 100}'
```
---
## For End Users
### Step 3: Validate Your API Key
Before making LLM calls, validate your key works by calling the `/v1/models` endpoint:
```bash
curl -X GET http://localhost:4000/v1/models \
-H "Authorization: Bearer <your-api-key>"
```
- If your key is valid, you'll get a list of available models.
- If invalid, you'll get a 401 error.
---
### Step 4: Hello World - Make Your First LLM Call
```bash
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer <your-api-key>" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Hello!"}]
}'
```
---
## Troubleshooting
- If you get a 401 error, check with your admin that your key is active and you have access to the requested model.
- Use the `/v1/models` endpoint to quickly check if your key is valid without consuming LLM tokens.
---
## See Also
- [Proxy Quick Start](./quick_start.md)
- [User Management](./users.md)
- [Key Management](./key_management.md)

2
docs/my-website/release_notes/v1.76.0-stable/index.md
View file

@ -1,5 +1,5 @@
---
title: "[PRE-RELEASE]v1.76.0-stable - RPS Improvements"
title: "v1.76.0-stable - RPS Improvements"
slug: "v1-76-0"
date: 2025-08-23T10:00:00
authors:

14
docs/my-website/release_notes/v1.77.2-stable/index.md
View file

@ -1,5 +1,5 @@
---
title: "[Pre-Release] v1.77.2-stable - Bedrock Batches API"
title: "v1.77.2-stable - Bedrock Batches API"
slug: "v1-77-2"
date: 2025-09-13T10:00:00
authors:
@ -21,22 +21,22 @@ import TabItem from '@theme/TabItem';
## Deploy this version
:::info
This release is not yet live.
:::
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.77.2-stable
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.77.2.post1
```
</TabItem>

258
docs/my-website/release_notes/v1.77.3-stable/index.md Normal file
View file

@ -0,0 +1,258 @@
---
title: "[Preview] v1.77.3-stable - Priority Based Rate Limiting"
slug: "v1-77-3"
date: 2025-09-21T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.77.3.rc.1
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
pip install litellm==1.77.3
```
</TabItem>
</Tabs>
---
## Key Highlights
- **+550 RPS Performance Improvements** - Optimizations in request handling and object initialization.
- **Priority Quota Reservation** - Proxy admins can now reserve TPM/RPM capacity for specific keys.
## Priority Quota Reservation
This release adds support for priority quota reservation. This allows **Proxy Admins** to reserve TPM/RPM capacity for keys based on metadata priority levels, ensuring critical production workloads get guaranteed access regardless of development traffic volume.
Get started [here](../../docs/proxy/dynamic_rate_limit#priority-quota-reservation)
<iframe width="700" height="500" src="https://www.loom.com/embed/1b54b93139ee415d959402cc0629f3f7" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| SambaNova | `sambanova/deepseek-v3.1` | 128K | $0.90 | $0.90 | Chat completions |
| SambaNova | `sambanova/gpt-oss-120b` | 128K | $0.72 | $0.72 | Chat completions |
| OVHCloud | Various models | Varies | Contact provider | Contact provider | Chat completions |
| CompactifAI | Various models | Varies | Contact provider | Contact provider | Chat completions |
| TwelveLabs | `twelvelabs/marengo-embed-2.7` | 32K | $0.12 | $0.00 | Embeddings |
#### Features
- **[OVHCloud AI Endpoints](../../docs/providers/ovhcloud)**
- New provider support with comprehensive model catalog - [PR #14494](https://github.com/BerriAI/litellm/pull/14494)
- **[CompactifAI](../../docs/providers/compactifai)**
- New provider integration - [PR #14532](https://github.com/BerriAI/litellm/pull/14532)
- **[SambaNova](../../docs/providers/sambanova)**
- Added DeepSeek v3.1 and GPT-OSS-120B models - [PR #14500](https://github.com/BerriAI/litellm/pull/14500)
- **[Bedrock](../../docs/providers/bedrock)**
- Cross-region inference profile cost calculation - [PR #14566](https://github.com/BerriAI/litellm/pull/14566)
- AWS external ID parameter support for authentication - [PR #14582](https://github.com/BerriAI/litellm/pull/14582)
- CountTokens API implementation - [PR #14557](https://github.com/BerriAI/litellm/pull/14557)
- Titan V2 encoding_format parameter support - [PR #14687](https://github.com/BerriAI/litellm/pull/14687)
- Nova Canvas image generation inference profiles - [PR #14578](https://github.com/BerriAI/litellm/pull/14578)
- Bedrock Batches API - batch processing support with file upload and request transformation - [PR #14618](https://github.com/BerriAI/litellm/pull/14618)
- Bedrock Twelve Labs embedding provider support - [PR #14697](https://github.com/BerriAI/litellm/pull/14697)
- **[Vertex AI](../../docs/providers/vertex)**
- Gemini labels field provider-aware filtering - [PR #14563](https://github.com/BerriAI/litellm/pull/14563)
- Gemini Batch API support - [PR #14733](https://github.com/BerriAI/litellm/pull/14733)
- **[Volcengine](../../docs/providers/volcengine)**
- Fixed thinking parameters when disabled - [PR #14569](https://github.com/BerriAI/litellm/pull/14569)
- **[Cohere](../../docs/providers/cohere)**
- Handle Generate API deprecation, default to chat endpoints - [PR #14676](https://github.com/BerriAI/litellm/pull/14676)
- **[TwelveLabs](../../docs/providers/twelvelabs)**
- Added Marengo Embed 2.7 embedding support - [PR #14674](https://github.com/BerriAI/litellm/pull/14674)
### Bug Fixes
- **[Bedrock](../../docs/providers/bedrock)**
- Empty arguments handling in tool call invocation - [PR #14583](https://github.com/BerriAI/litellm/pull/14583)
- **[Vertex AI](../../docs/providers/vertex)**
- Avoid deepcopy crash with non-pickleables in Gemini/Vertex - [PR #14418](https://github.com/BerriAI/litellm/pull/14418)
- **[XAI](../../docs/providers/xai)**
- Fix unsupported stop parameter for grok-code models - [PR #14565](https://github.com/BerriAI/litellm/pull/14565)
- **[Gemini](../../docs/providers/gemini)**
- Updated error message for Gemini API - [PR #14589](https://github.com/BerriAI/litellm/pull/14589)
- Fixed 2.5 Flash Image Preview model routing - [PR #14715](https://github.com/BerriAI/litellm/pull/14715)
- API key passing for token counting endpoints - [PR #14744](https://github.com/BerriAI/litellm/pull/14744)
#### New Provider Support
- **[OVHCloud AI Endpoints](../../docs/providers/ovhcloud)**
- Complete provider integration with model catalog and authentication - [PR #14494](https://github.com/BerriAI/litellm/pull/14494)
- **[CompactifAI](../../docs/providers/compactifai)**
- New provider support with documentation - [PR #14532](https://github.com/BerriAI/litellm/pull/14532)
---
## LLM API Endpoints
#### Features
- **[/responses](../../docs/response_api)**
- Added cancel endpoint support for non-admin users - [PR #14594](https://github.com/BerriAI/litellm/pull/14594)
- Improved response session handling and cold storage configuration with s3 - [PR #14534](https://github.com/BerriAI/litellm/pull/14534)
- Added OpenAI & Azure /responses/cancel endpoint support - [PR #14561](https://github.com/BerriAI/litellm/pull/14561)
- **General**
- Enhanced rate limit error messages with details - [PR #14736](https://github.com/BerriAI/litellm/pull/14736)
- Middle-truncation for spend log payloads - [PR #14637](https://github.com/BerriAI/litellm/pull/14637)
#### Bugs
- **[/chat/completions](../../docs/completion/input)**
- Fixed completion chat ID handling - [PR #14548](https://github.com/BerriAI/litellm/pull/14548)
- Prevent AttributeError for _get_tags_from_request_kwargs - [PR #14735](https://github.com/BerriAI/litellm/pull/14735)
- **[/responses](../../docs/response_api)**
- Fixed cost calculation - [PR #14675](https://github.com/BerriAI/litellm/pull/14675)
- **General**
- Rate limiter AttributeError fix - [PR #14609](https://github.com/BerriAI/litellm/pull/14609)
---
## Spend Tracking, Budgets and Rate Limiting
- **Responses API Cost Calculation** fix - [PR #14675](https://github.com/BerriAI/litellm/pull/14675)
- **Anthropic Cache Token Pricing** - Separate 1-hour vs 5-minute cache creation costs - [PR #14620](https://github.com/BerriAI/litellm/pull/14620), [PR #14652](https://github.com/BerriAI/litellm/pull/14652)
- **Indochina Time Timezone** support for budget resets - [PR #14666](https://github.com/BerriAI/litellm/pull/14666)
- **Soft Budget Alert Cache Issues** - Resolved soft budget alert cache issues - [PR #14491](https://github.com/BerriAI/litellm/pull/14491)
- **Dynamic Rate Limiter v3** - Priority routing improvements - [PR #14734](https://github.com/BerriAI/litellm/pull/14734)
- **Enhanced Rate Limit Errors** - More detailed error messages - [PR #14736](https://github.com/BerriAI/litellm/pull/14736)
---
## Management Endpoints / UI
#### Features
- **Team Member Service Account Keys** - Allow team members to view keys they create - [PR #14619](https://github.com/BerriAI/litellm/pull/14619)
- **Default Budget for JWT Teams** - Auto-assign budgets to generated teams - [PR #14514](https://github.com/BerriAI/litellm/pull/14514)
- **SSO Access Control Groups** - Enhanced token info endpoint integration - [PR #14738](https://github.com/BerriAI/litellm/pull/14738)
- **Health Test Connect Protection** - Restrict access based on model creation permissions - [PR #14650](https://github.com/BerriAI/litellm/pull/14650)
- **Amazon Bedrock Guardrail Info View** - Enhanced logging visualization - [PR #14696](https://github.com/BerriAI/litellm/pull/14696)
#### Bug Fixes
- **SCIM v2** - Fix group PUSH and PUT operations for non-existent members - [PR #14581](https://github.com/BerriAI/litellm/pull/14581)
- **Guardrail View/Edit/Delete** behavior fixes - [PR #14622](https://github.com/BerriAI/litellm/pull/14622)
- **In-Memory Guardrail** update failures - [PR #14653](https://github.com/BerriAI/litellm/pull/14653)
---
## Logging / Guardrail Integrations
#### Features
- **[DataDog](../../docs/proxy/logging#datadog)**
- Enhanced spend tracking metrics - [PR #14555](https://github.com/BerriAI/litellm/pull/14555)
- Stream support with is_streamed_request parameter - [PR #14673](https://github.com/BerriAI/litellm/pull/14673)
- Fixed tool calls metadata passing - [PR #14531](https://github.com/BerriAI/litellm/pull/14531)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Added logging support for Responses API - [PR #14597](https://github.com/BerriAI/litellm/pull/14597)
- **[Langsmith](../../docs/proxy/logging#langsmith)**
- Langsmith Sampling Rate - Key/Team-level tracing configuration - [PR #14740](https://github.com/BerriAI/litellm/pull/14740)
- **[Prometheus](../../docs/proxy/logging#prometheus)**
- Multi-worker support improvements - [PR #14530](https://github.com/BerriAI/litellm/pull/14530)
- User email labels in monitoring - [PR #14520](https://github.com/BerriAI/litellm/pull/14520)
- **[Opik](../../docs/proxy/logging#opik)**
- Fixed timezone issue - [PR #14708](https://github.com/BerriAI/litellm/pull/14708)
### Bug Fixes
- **[S3](../../docs/proxy/logging#s3-buckets)**
- Fixed 404 error when using s3_endpoint_url - [PR #14559](https://github.com/BerriAI/litellm/pull/14559)
#### Guardrails
- **Tool Permission Guardrail** - Fine-grained tool access control - [PR #14519](https://github.com/BerriAI/litellm/pull/14519)
- **Bedrock Guardrails** - Selective guarding support with runtime endpoint configuration - [PR #14575](https://github.com/BerriAI/litellm/pull/14575), [PR #14650](https://github.com/BerriAI/litellm/pull/14650)
- **Default Last Message** in guardrails - [PR #14640](https://github.com/BerriAI/litellm/pull/14640)
- **AWS exceptions handling despite 200 response** - [PR #14658](https://github.com/BerriAI/litellm/pull/14658)
#### New Integration
- **[PostHog](../../docs/observability/posthog)** - Complete observability integration for LiteLLM usage tracking and analytics - [PR #14610](https://github.com/BerriAI/litellm/pull/14610)
---
## MCP Gateway
- **MCP Server Alias Parsing** - Multi-part URL path support - [PR #14558](https://github.com/BerriAI/litellm/pull/14558)
- **MCP Filter Recomputation** - After server deletion - [PR #14542](https://github.com/BerriAI/litellm/pull/14542)
- **MCP Gateway Tools List** improvements - [PR #14695](https://github.com/BerriAI/litellm/pull/14695)
---
## Performance / Loadbalancing / Reliability improvements
- **+500 RPS Performance Boost** when sending the `user` field - [PR #14616](https://github.com/BerriAI/litellm/pull/14616)
- **+50 RPS** by removing iscoroutine from hot path - [PR #14649](https://github.com/BerriAI/litellm/pull/14649)
- **7% reduction** in __init__ overhead - [PR #14689](https://github.com/BerriAI/litellm/pull/14689)
- **Generic Object Pool** implementation for better resource management - [PR #14702](https://github.com/BerriAI/litellm/pull/14702)
---
## General Proxy Improvements
- **Middle-Truncation** for spend log payloads - [PR #14637](https://github.com/BerriAI/litellm/pull/14637)
#### Security
- **Security Update** - Bump aiohttp==3.12.14, fix CVE-2025-53643 - [PR #14638](https://github.com/BerriAI/litellm/pull/14638)
---
## New Contributors
* @luisfucros made their first contribution in [PR #14500](https://github.com/BerriAI/litellm/pull/14500)
* @hanakannzashi made their first contribution in [PR #14548](https://github.com/BerriAI/litellm/pull/14548)
* @eliasto made their first contribution in [PR #14494](https://github.com/BerriAI/litellm/pull/14494)
* @Rasmusafj made their first contribution in [PR #14491](https://github.com/BerriAI/litellm/pull/14491)
* @LingXuanYin made their first contribution in [PR #14569](https://github.com/BerriAI/litellm/pull/14569)
* @ronaldpereira made their first contribution in [PR #14613](https://github.com/BerriAI/litellm/pull/14613)
* @hula-la made their first contribution in [PR #14534](https://github.com/BerriAI/litellm/pull/14534)
* @carlos-marchal-ph made their first contribution in [PR #14610](https://github.com/BerriAI/litellm/pull/14610)
* @akraines made their first contribution in [PR #14637](https://github.com/BerriAI/litellm/pull/14637)
* @mrFranklin made their first contribution in [PR #14708](https://github.com/BerriAI/litellm/pull/14708)
* @tcx4c70 made their first contribution in [PR #14675](https://github.com/BerriAI/litellm/pull/14675)
* @michaeltansg made their first contribution in [PR #14666](https://github.com/BerriAI/litellm/pull/14666)
* @tosi29 made their first contribution in [PR #14725](https://github.com/BerriAI/litellm/pull/14725)
* @gmdfalk made their first contribution in [PR #14735](https://github.com/BerriAI/litellm/pull/14735)
* @FelipeRodriguesGare made their first contribution in [PR #14733](https://github.com/BerriAI/litellm/pull/14733)
* @mritunjaysharma394 made their first contribution in [PR #14678](https://github.com/BerriAI/litellm/pull/14678)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.77.2.rc.1...v1.77.3.rc.1)**

163
docs/my-website/sidebars.js
View file

@ -75,6 +75,7 @@ const sidebars = {
type: "category",
label: "AI Tools (OpenWebUI, Claude Code, etc.)",
items: [
"integrations/letta",
"tutorials/openweb_ui",
"tutorials/openai_codex",
"tutorials/litellm_gemini_cli",
@ -111,6 +112,7 @@ const sidebars = {
label: "Setup & Deployment",
items: [
"proxy/quick_start",
"proxy/user_onboarding",
"proxy/deploy",
"proxy/prod",
"proxy/cli",
@ -132,7 +134,6 @@ const sidebars = {
label: "All Endpoints (Swagger)",
href: "https://litellm-api.up.railway.app/",
},
"proxy/enterprise",
"proxy/management_cli",
{
type: "category",
@ -154,11 +155,9 @@ const sidebars = {
"proxy/token_auth",
"proxy/service_accounts",
"proxy/access_control",
"proxy/cli_sso",
"proxy/custom_auth",
"proxy/ip_address",
"proxy/email",
"proxy/multiple_admins",
"proxy/custom_auth",
],
},
{
@ -169,30 +168,6 @@ const sidebars = {
"proxy/team_model_add"
]
},
{
type: "category",
label: "Admin UI",
items: [
"proxy/ui",
"proxy/admin_ui_sso",
"proxy/custom_root_ui",
"proxy/model_hub",
"proxy/self_serve",
"proxy/public_teams",
"tutorials/scim_litellm",
"proxy/custom_sso",
"proxy/ui_credentials",
"proxy/ui/bulk_edit_users",
{
type: "category",
label: "UI Logs",
items: [
"proxy/ui_logs",
"proxy/ui_logs_sessions"
]
}
],
},
{
type: "category",
label: "Spend Tracking",
@ -201,7 +176,48 @@ const sidebars = {
{
type: "category",
label: "Budgets + Rate Limits",
items: ["proxy/users", "proxy/temporary_budget_increase", "proxy/rate_limit_tiers", "proxy/team_budgets", "proxy/customers"],
items: ["proxy/users", "proxy/temporary_budget_increase", "proxy/rate_limit_tiers", "proxy/team_budgets", "proxy/dynamic_rate_limit", "proxy/customers"],
},
{
type: "category",
label: "Enterprise Features",
items: [
"proxy/enterprise",
{
type: "category",
label: "Admin UI",
items: [
"proxy/ui",
"proxy/admin_ui_sso",
"proxy/custom_root_ui",
"proxy/model_hub",
"proxy/self_serve",
"proxy/public_teams",
"proxy/ui_credentials",
"proxy/ui/bulk_edit_users",
{
type: "category",
label: "UI Logs",
items: [
"proxy/ui_logs",
"proxy/ui_logs_sessions"
]
}
],
},
{
type: "category",
label: "SSO & Identity Management",
items: [
"proxy/cli_sso",
"proxy/admin_ui_sso",
"proxy/custom_sso",
"tutorials/scim_litellm",
"tutorials/msft_sso",
"proxy/multiple_admins",
],
},
],
},
{
type: "link",
@ -250,6 +266,25 @@ const sidebars = {
slug: "/supported_endpoints",
},
items: [
"anthropic_unified",
"apply_guardrail",
"assistants",
{
type: "category",
label: "/audio",
"items": [
"audio_transcription",
"text_to_speech",
]
},
{
type: "category",
label: "/batches",
items: [
"batches",
"proxy/managed_batches",
]
},
{
type: "category",
label: "/chat/completions",
@ -266,11 +301,23 @@ const sidebars = {
"completion/http_handler_config",
],
},
"response_api",
"text_completion",
"embedding/supported_embedding",
"anthropic_unified",
"mcp",
{
type: "category",
label: "/files",
items: [
"files_endpoints",
"proxy/litellm_managed_files",
],
},
{
type: "category",
label: "/fine_tuning",
items: [
"fine_tuning",
"proxy/managed_finetuning",
]
},
"generateContent",
{
type: "category",
@ -281,21 +328,8 @@ const sidebars = {
"image_variations",
]
},
{
type: "category",
label: "/audio",
"items": [
"audio_transcription",
"text_to_speech",
]
},
{
type: "category",
label: "/vector_stores",
items: [
"vector_stores/search",
]
},
"mcp",
"moderation",
{
type: "category",
label: "Pass-through Endpoints (Anthropic SDK, etc.)",
@ -314,36 +348,17 @@ const sidebars = {
"proxy/pass_through",
],
},
"rerank",
"assistants",
{
type: "category",
label: "/files",
items: [
"files_endpoints",
"proxy/litellm_managed_files",
],
},
{
type: "category",
label: "/batches",
items: [
"batches",
"proxy/managed_batches",
]
},
"realtime",
"rerank",
"response_api",
"text_completion",
{
type: "category",
label: "/fine_tuning",
label: "/vector_stores",
items: [
"fine_tuning",
"proxy/managed_finetuning",
"vector_stores/search",
]
},
"moderation",
"apply_guardrail",
],
},
{
@ -383,6 +398,7 @@ const sidebars = {
items: [
"providers/azure_ai",
"providers/azure_ai_img",
"providers/azure_ai_img_edit",
]
},
{
@ -392,6 +408,7 @@ const sidebars = {
"providers/vertex",
"providers/vertex_partner",
"providers/vertex_image",
"providers/vertex_batch",
]
},
{
@ -523,6 +540,7 @@ const sidebars = {
"completion/batching",
"completion/mock_requests",
"completion/reliable_completions",
"proxy/veo_video_generation",
]
},
@ -544,6 +562,7 @@ const sidebars = {
items: [
"set_keys",
"completion/token_usage",
"sdk/headers",
"sdk_custom_pricing",
"embedding/async_embedding",
"embedding/moderation",

1
enterprise/litellm_enterprise/integrations/prometheus.py
View file

@ -2328,7 +2328,6 @@ def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]:
"tag_Service_web_app_v1": "false",
}
"""
import re
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
from litellm.types.integrations.prometheus import _sanitize_prometheus_label_name

1
enterprise/litellm_enterprise/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -2,7 +2,6 @@
Enterprise internal user management endpoints
"""
import os
from fastapi import APIRouter, Depends, HTTPException

2
enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py
View file

@ -11,7 +11,7 @@ All /vector_store management endpoints
import copy
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from fastapi import APIRouter, Depends, HTTPException
import litellm
from litellm._logging import verbose_proxy_logger

0
git_model_armor.py Normal file
View file

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.19-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.19.tar.gz vendored Normal file
View file

Binary file not shown.

8
litellm-proxy-extras/litellm_proxy_extras/migrations/20250918083359_drop_spec_version_column_from_mcp_table/migration.sql Normal file
View file

@ -0,0 +1,8 @@
/*
Warnings:
- You are about to drop the column `spec_version` on the `LiteLLM_MCPServerTable` table. All the data in the column will be lost.
*/
-- AlterTable
ALTER TABLE "public"."LiteLLM_MCPServerTable" DROP COLUMN "spec_version";

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -171,7 +171,6 @@ model LiteLLM_MCPServerTable {
description String?
url String?
transport String @default("sse")
spec_version String @default("2025-03-26")
auth_type String?
created_at DateTime? @default(now()) @map("created_at")
created_by String?

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.2.18"
version = "0.2.19"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.2.18"
version = "0.2.19"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

14
litellm/__init__.py
View file

@ -60,6 +60,7 @@ from litellm.constants import (
empower_models,
together_ai_models,
baseten_models,
WANDB_MODELS,
REPEATED_STREAMING_CHUNK_LIMIT,
request_timeout,
open_ai_embedding_models,
@ -88,6 +89,7 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
LiteLLM_UpperboundKeyGenerateParams,
)
from litellm.types.utils import StandardKeyGenerationConfig, LlmProviders
from litellm.types.utils import PriorityReservationSettings
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
import httpx
@ -117,6 +119,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"logfire",
"literalai",
"dynamic_rate_limiter",
"dynamic_rate_limiter_v3",
"langsmith",
"prometheus",
"otel",
@ -241,6 +244,7 @@ novita_api_key: Optional[str] = None
snowflake_key: Optional[str] = None
gradient_ai_api_key: Optional[str] = None
nebius_key: Optional[str] = None
wandb_key: Optional[str] = None
heroku_key: Optional[str] = None
cometapi_key: Optional[str] = None
ovhcloud_key: Optional[str] = None
@ -370,6 +374,7 @@ public_model_groups: Optional[List[str]] = None
public_model_groups_links: Dict[str, str] = {}
#### REQUEST PRIORITIZATION ######
priority_reservation: Optional[Dict[str, float]] = None
priority_reservation_settings: "PriorityReservationSettings" = PriorityReservationSettings()
######## Networking Settings ########
@ -523,6 +528,7 @@ cometapi_models: Set = set()
oci_models: Set = set()
vercel_ai_gateway_models: Set = set()
volcengine_models: Set = set()
wandb_models: Set = set(WANDB_MODELS)
ovhcloud_models: Set = set()
ovhcloud_embedding_models: Set = set()
@ -739,6 +745,8 @@ def add_known_models():
oci_models.add(key)
elif value.get("litellm_provider") == "volcengine":
volcengine_models.add(key)
elif value.get("litellm_provider") == "wandb":
wandb_models.add(key)
elif value.get("litellm_provider") == "ovhcloud":
ovhcloud_models.add(key)
elif value.get("litellm_provider") == "ovhcloud-embedding-models":
@ -837,6 +845,7 @@ model_list = list(
| heroku_models
| vercel_ai_gateway_models
| volcengine_models
| wandb_models
| ovhcloud_models
)
@ -919,6 +928,7 @@ models_by_provider: dict = {
"cometapi": cometapi_models,
"oci": oci_models,
"volcengine": volcengine_models,
"wandb": wandb_models,
"ovhcloud": ovhcloud_models | ovhcloud_embedding_models,
}
@ -1258,6 +1268,7 @@ from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig
from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig
from .llms.github_copilot.chat.transformation import GithubCopilotConfig
from .llms.nebius.chat.transformation import NebiusConfig
from .llms.wandb.chat.transformation import WandbConfig
from .llms.dashscope.chat.transformation import DashScopeChatConfig
from .llms.moonshot.chat.transformation import MoonshotChatConfig
from .llms.v0.chat.transformation import V0ChatConfig
@ -1334,5 +1345,8 @@ disable_hf_tokenizer_download: Optional[bool] = (
)
global_disable_no_log_param: bool = False
### CLI UTILITIES ###
from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
### PASSTHROUGH ###
from .passthrough import allm_passthrough_route, llm_passthrough_route

23
litellm/_uuid.py Normal file
View file

@ -0,0 +1,23 @@
"""
Internal unified UUID helper.
Tries to use fastuuid (performance) and falls back to stdlib uuid if unavailable.
"""
FASTUUID_AVAILABLE = False
try:
import fastuuid as _uuid # type: ignore
FASTUUID_AVAILABLE = True
except Exception: # pragma: no cover - fallback path
import uuid as _uuid # type: ignore
# Expose a module-like alias so callers can use: uuid.uuid4()
uuid = _uuid
def uuid4():
"""Return a UUID4 using the selected backend."""
return uuid.uuid4()

37
litellm/constants.py
View file

@ -313,6 +313,7 @@ LITELLM_CHAT_PROVIDERS = [
"morph",
"lambda_ai",
"vercel_ai_gateway",
"wandb",
"ovhcloud",
]
@ -448,6 +449,7 @@ openai_compatible_endpoints: List = [
"https://api.lambda.ai/v1",
"https://api.hyperbolic.xyz/v1",
"https://ai-gateway.vercel.sh/v1",
"https://api.inference.wandb.ai/v1",
]
@ -492,6 +494,7 @@ openai_compatible_providers: List = [
"hyperbolic",
"vercel_ai_gateway",
"aiml",
"wandb",
]
openai_text_completion_compatible_providers: List = (
[ # providers that support `/v1/completions`
@ -507,6 +510,7 @@ openai_text_completion_compatible_providers: List = (
"v0",
"lambda_ai",
"hyperbolic",
"wandb",
]
)
_openai_like_providers: List = [
@ -757,6 +761,38 @@ nebius_embedding_models: set = set(
]
)
WANDB_MODELS: set = set(
[
# openai models
"openai/gpt-oss-120b",
"openai/gpt-oss-20b",
# zai-org models
"zai-org/GLM-4.5",
# Qwen models
"Qwen/Qwen3-235B-A22B-Instruct-2507",
"Qwen/Qwen3-Coder-480B-A35B-Instruct",
"Qwen/Qwen3-235B-A22B-Thinking-2507",
# moonshotai
"moonshotai/Kimi-K2-Instruct",
# meta models
"meta-llama/Llama-3.1-8B-Instruct",
"meta-llama/Llama-3.3-70B-Instruct",
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
# deepseek-ai
"deepseek-ai/DeepSeek-V3.1",
"deepseek-ai/DeepSeek-R1-0528",
"deepseek-ai/DeepSeek-V3-0324",
# microsoft
"microsoft/Phi-4-mini-instruct",
]
)
BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"cohere",
"anthropic",
@ -947,6 +983,7 @@ HEALTH_CHECK_TIMEOUT_SECONDS = int(
os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60)
) # 60 seconds
LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check"
LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli"
UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard"
LITELLM_PROXY_ADMIN_NAME = "default_user_id"

15
litellm/cost_calculator.py
View file

@ -148,6 +148,8 @@ def cost_per_token( # noqa: PLR0915
### CALL TYPE ###
call_type: CallTypesLiteral = "completion",
audio_transcription_file_duration: float = 0.0, # for audio transcription calls - the file time in seconds
### SERVICE TIER ###
service_tier: Optional[str] = None, # for OpenAI service tier pricing
) -> Tuple[float, float]: # type: ignore
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@ -278,6 +280,7 @@ def cost_per_token( # noqa: PLR0915
model=model_without_prefix,
usage=usage_block,
custom_llm_provider=custom_llm_provider,
service_tier=service_tier,
)
return prompt_cost, completion_cost
@ -327,7 +330,7 @@ def cost_per_token( # noqa: PLR0915
elif custom_llm_provider == "bedrock":
return bedrock_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "openai":
return openai_cost_per_token(model=model, usage=usage_block)
return openai_cost_per_token(model=model, usage=usage_block, service_tier=service_tier)
elif custom_llm_provider == "databricks":
return databricks_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "fireworks_ai":
@ -606,6 +609,8 @@ def completion_cost( # noqa: PLR0915
litellm_model_name: Optional[str] = None,
router_model_id: Optional[str] = None,
litellm_logging_obj: Optional[LitellmLoggingObject] = None,
### SERVICE TIER ###
service_tier: Optional[str] = None, # for OpenAI service tier pricing
) -> float:
"""
Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm.
@ -658,6 +663,10 @@ def completion_cost( # noqa: PLR0915
completion_response=completion_response
)
rerank_billed_units: Optional[RerankBilledUnits] = None
# Extract service_tier from optional_params if not provided directly
if service_tier is None and optional_params is not None:
service_tier = optional_params.get("service_tier")
selected_model = _select_model_name_for_cost_calc(
model=model,
@ -909,6 +918,7 @@ def completion_cost( # noqa: PLR0915
call_type=cast(CallTypesLiteral, call_type),
audio_transcription_file_duration=audio_transcription_file_duration,
rerank_billed_units=rerank_billed_units,
service_tier=service_tier,
)
_final_cost = (
prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar
@ -1003,6 +1013,8 @@ def response_cost_calculator(
litellm_model_name: Optional[str] = None,
router_model_id: Optional[str] = None,
litellm_logging_obj: Optional[LitellmLoggingObject] = None,
### SERVICE TIER ###
service_tier: Optional[str] = None, # for OpenAI service tier pricing
) -> float:
"""
Returns
@ -1036,6 +1048,7 @@ def response_cost_calculator(
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
litellm_logging_obj=litellm_logging_obj,
service_tier=service_tier,
)
return response_cost
except Exception as e:

59
litellm/experimental_mcp_client/client.py
View file

@ -19,8 +19,6 @@ from litellm._logging import verbose_logger
from litellm.types.mcp import (
MCPAuth,
MCPAuthType,
MCPSpecVersion,
MCPSpecVersionType,
MCPStdioConfig,
MCPTransport,
MCPTransportType,
@ -48,7 +46,6 @@ class MCPClient:
auth_value: Optional[str] = None,
timeout: float = 60.0,
stdio_config: Optional[MCPStdioConfig] = None,
protocol_version: MCPSpecVersionType = MCPSpecVersion.jun_2025,
):
self.server_url: str = server_url
self.transport_type: MCPTransport = transport_type
@ -62,7 +59,6 @@ class MCPClient:
self._session_ctx = None
self._task: Optional[asyncio.Task] = None
self.stdio_config: Optional[MCPStdioConfig] = stdio_config
self.protocol_version: MCPSpecVersionType = protocol_version
# handle the basic auth value if provided
if auth_value:
@ -84,22 +80,24 @@ class MCPClient:
"""Initialize the transport and session."""
if self._session:
return # Already connected
try:
if self.transport_type == MCPTransport.stdio:
# For stdio transport, use stdio_client with command-line parameters
if not self.stdio_config:
raise ValueError("stdio_config is required for stdio transport")
server_params = StdioServerParameters(
command=self.stdio_config.get("command", ""),
args=self.stdio_config.get("args", []),
env=self.stdio_config.get("env", {})
env=self.stdio_config.get("env", {}),
)
self._transport_ctx = stdio_client(server_params)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(self._transport[0], self._transport[1])
self._session_ctx = ClientSession(
self._transport[0], self._transport[1]
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
elif self.transport_type == MCPTransport.sse:
@ -110,7 +108,9 @@ class MCPClient:
headers=headers,
)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(self._transport[0], self._transport[1])
self._session_ctx = ClientSession(
self._transport[0], self._transport[1]
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
else: # http
@ -121,7 +121,9 @@ class MCPClient:
headers=headers,
)
self._transport = await self._transport_ctx.__aenter__()
self._session_ctx = ClientSession(self._transport[0], self._transport[1])
self._session_ctx = ClientSession(
self._transport[0], self._transport[1]
)
self._session = await self._session_ctx.__aenter__()
await self._session.initialize()
except ValueError as e:
@ -184,8 +186,10 @@ class MCPClient:
def _get_auth_headers(self) -> dict:
"""Generate authentication headers based on auth type."""
headers = {}
headers = {
"MCP-Protocol-Version": "2025-06-18"
}
if self._mcp_auth_value:
if self.auth_type == MCPAuth.bearer_token:
headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
@ -196,18 +200,8 @@ class MCPClient:
elif self.auth_type == MCPAuth.authorization:
headers["Authorization"] = self._mcp_auth_value
# Handle protocol version - it might be a string or enum
if hasattr(self.protocol_version, 'value'):
# It's an enum
protocol_version_str = self.protocol_version.value
else:
# It's a string
protocol_version_str = str(self.protocol_version)
headers["MCP-Protocol-Version"] = protocol_version_str
return headers
async def list_tools(self) -> List[MCPTool]:
"""List available tools from the server."""
if not self._session:
@ -216,7 +210,7 @@ class MCPClient:
except Exception as e:
verbose_logger.warning(f"MCP client connection failed: {str(e)}")
return []
if self._session is None:
verbose_logger.warning("MCP client session is not initialized")
return []
@ -245,17 +239,20 @@ class MCPClient:
except Exception as e:
verbose_logger.warning(f"MCP client connection failed: {str(e)}")
return MCPCallToolResult(
content=[TextContent(type="text", text=f"{str(e)}")],
isError=True
content=[TextContent(type="text", text=f"{str(e)}")], isError=True
)
if self._session is None:
verbose_logger.warning("MCP client session is not initialized")
return MCPCallToolResult(
content=[TextContent(type="text", text="MCP client session is not initialized")],
content=[
TextContent(
type="text", text="MCP client session is not initialized"
)
],
isError=True,
)
try:
tool_result = await self._session.call_tool(
name=call_tool_request_params.name,
@ -270,8 +267,8 @@ class MCPClient:
await self.disconnect()
# Return a default error result instead of raising
return MCPCallToolResult(
content=[TextContent(type="text", text=f"{str(e)}")], # Empty content for error case
content=[
TextContent(type="text", text=f"{str(e)}")
], # Empty content for error case
isError=True,
)

28
litellm/files/main.py
View file

@ -731,7 +731,7 @@ def file_list(
async def afile_content(
file_id: str,
custom_llm_provider: Literal["openai", "azure"] = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
extra_headers: Optional[Dict[str, str]] = None,
extra_body: Optional[Dict[str, str]] = None,
**kwargs,
@ -887,6 +887,32 @@ def file_content(
client=client,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "vertex_ai":
api_base = optional_params.api_base or ""
vertex_ai_project = (
optional_params.vertex_project
or litellm.vertex_project
or get_secret_str("VERTEXAI_PROJECT")
)
vertex_ai_location = (
optional_params.vertex_location
or litellm.vertex_location
or get_secret_str("VERTEXAI_LOCATION")
)
vertex_credentials = optional_params.vertex_credentials or get_secret_str(
"VERTEXAI_CREDENTIALS"
)
response = vertex_ai_files_instance.file_content(
_is_async=_is_async,
file_content_request=_file_content_request,
api_base=api_base,
vertex_credentials=vertex_credentials,
vertex_project=vertex_ai_project,
vertex_location=vertex_ai_location,
timeout=timeout,
max_retries=optional_params.max_retries,
)
else:
raise litellm.exceptions.BadRequestError(
message="LiteLLM doesn't support {} for 'custom_llm_provider'. Supported providers are 'openai', 'azure', 'vertex_ai'.".format(

58
litellm/integrations/langsmith.py
View file

@ -39,6 +39,7 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_api_key: Optional[str] = None,
langsmith_project: Optional[str] = None,
langsmith_base_url: Optional[str] = None,
langsmith_sampling_rate: Optional[float] = None,
**kwargs,
):
self.flush_lock = asyncio.Lock()
@ -49,7 +50,8 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_base_url=langsmith_base_url,
)
self.sampling_rate: float = (
float(os.getenv("LANGSMITH_SAMPLING_RATE")) # type: ignore
langsmith_sampling_rate
or float(os.getenv("LANGSMITH_SAMPLING_RATE")) # type: ignore
if os.getenv("LANGSMITH_SAMPLING_RATE") is not None
and os.getenv("LANGSMITH_SAMPLING_RATE").strip().isdigit() # type: ignore
else 1.0
@ -76,26 +78,14 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_base_url: Optional[str] = None,
) -> LangsmithCredentialsObject:
_credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
if _credentials_api_key is None:
raise Exception(
"Invalid Langsmith API Key given. _credentials_api_key=None."
)
_credentials_project = (
langsmith_project or os.getenv("LANGSMITH_PROJECT") or "litellm-completion"
)
if _credentials_project is None:
raise Exception(
"Invalid Langsmith API Key given. _credentials_project=None."
)
_credentials_base_url = (
langsmith_base_url
or os.getenv("LANGSMITH_BASE_URL")
or "https://api.smith.langchain.com"
)
if _credentials_base_url is None:
raise Exception(
"Invalid Langsmith API Key given. _credentials_base_url=None."
)
return LangsmithCredentialsObject(
LANGSMITH_API_KEY=_credentials_api_key,
@ -200,12 +190,7 @@ class LangsmithLogger(CustomBatchLogger):
def log_success_event(self, kwargs, response_obj, start_time, end_time):
try:
sampling_rate = (
float(os.getenv("LANGSMITH_SAMPLING_RATE")) # type: ignore
if os.getenv("LANGSMITH_SAMPLING_RATE") is not None
and os.getenv("LANGSMITH_SAMPLING_RATE").strip().isdigit() # type: ignore
else 1.0
)
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
random_sample = random.random()
if random_sample > sampling_rate:
verbose_logger.info(
@ -219,6 +204,7 @@ class LangsmithLogger(CustomBatchLogger):
kwargs,
response_obj,
)
credentials = self._get_credentials_to_use_for_request(kwargs=kwargs)
data = self._prepare_log_data(
kwargs=kwargs,
@ -245,7 +231,7 @@ class LangsmithLogger(CustomBatchLogger):
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
try:
sampling_rate = self.sampling_rate
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
random_sample = random.random()
if random_sample > sampling_rate:
verbose_logger.info(
@ -286,7 +272,7 @@ class LangsmithLogger(CustomBatchLogger):
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
sampling_rate = self.sampling_rate
sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
random_sample = random.random()
if random_sample > sampling_rate:
verbose_logger.info(
@ -417,6 +403,17 @@ class LangsmithLogger(CustomBatchLogger):
for queue_object in self.log_queue:
credentials = queue_object["credentials"]
# if credential missing, skip - log warning
if (
credentials["LANGSMITH_API_KEY"] is None
or credentials["LANGSMITH_PROJECT"] is None
):
verbose_logger.warning(
"Langsmith Logging - credentials missing - api_key: %s, project: %s",
credentials["LANGSMITH_API_KEY"],
credentials["LANGSMITH_PROJECT"],
)
continue
key = CredentialsKey(
api_key=credentials["LANGSMITH_API_KEY"],
project=credentials["LANGSMITH_PROJECT"],
@ -432,6 +429,19 @@ class LangsmithLogger(CustomBatchLogger):
return log_queue_by_credentials
def _get_sampling_rate_to_use_for_request(self, kwargs: Dict[str, Any]) -> float:
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
sampling_rate: float = self.sampling_rate
if standard_callback_dynamic_params is not None:
_sampling_rate = standard_callback_dynamic_params.get(
"langsmith_sampling_rate"
)
if _sampling_rate is not None:
sampling_rate = float(_sampling_rate)
return sampling_rate
def _get_credentials_to_use_for_request(
self, kwargs: Dict[str, Any]
) -> LangsmithCredentialsObject:
@ -442,9 +452,9 @@ class LangsmithLogger(CustomBatchLogger):
Otherwise, use the default credentials.
"""
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params", None)
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
if standard_callback_dynamic_params is not None:
credentials = self.get_credentials_from_env(
langsmith_api_key=standard_callback_dynamic_params.get(

9
litellm/integrations/opik/opik.py
View file

@ -3,6 +3,7 @@ Opik Logger that logs LLM events to an Opik server
"""
import asyncio
from datetime import timezone
import json
import traceback
from typing import Dict, List
@ -291,8 +292,8 @@ class OpikLogger(CustomBatchLogger):
"project_name": project_name,
"id": trace_id,
"name": trace_name,
"start_time": start_time.isoformat() + "Z",
"end_time": end_time.isoformat() + "Z",
"start_time": start_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
"end_time": end_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
"input": input_data,
"output": output_data,
"metadata": metadata,
@ -312,8 +313,8 @@ class OpikLogger(CustomBatchLogger):
"parent_span_id": parent_span_id,
"name": span_name,
"type": "llm",
"start_time": start_time.isoformat() + "Z",
"end_time": end_time.isoformat() + "Z",
"start_time": start_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
"end_time": end_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
"input": input_data,
"output": output_data,
"metadata": metadata,

58
litellm/litellm_core_utils/cli_token_utils.py Normal file
View file

@ -0,0 +1,58 @@
"""
CLI Token Utilities
SDK-level utilities for reading CLI authentication tokens.
This module has no dependencies on proxy code and can be safely imported at the SDK level.
"""
import json
import os
from pathlib import Path
from typing import Optional
def get_cli_token_file_path() -> str:
"""Get the path to the CLI token file"""
home_dir = Path.home()
config_dir = home_dir / ".litellm"
return str(config_dir / "token.json")
def load_cli_token() -> Optional[dict]:
"""Load CLI token data from file"""
token_file = get_cli_token_file_path()
if not os.path.exists(token_file):
return None
try:
with open(token_file, 'r') as f:
return json.load(f)
except (json.JSONDecodeError, IOError):
return None
def get_litellm_gateway_api_key() -> Optional[str]:
"""
Get the stored CLI API key for use with LiteLLM SDK.
This function reads the token file created by `litellm-proxy login`
and returns the API key for use in Python scripts.
Returns:
str: The API key if found, None otherwise
Example:
>>> import litellm
>>> api_key = litellm.get_litellm_gateway_api_key()
>>> if api_key:
>>> response = litellm.completion(
>>> model="gpt-3.5-turbo",
>>> messages=[{"role": "user", "content": "Hello"}],
>>> api_key=api_key,
>>> base_url="https://your-proxy.com/v1"
>>> )
"""
token_data = load_cli_token()
if token_data and 'key' in token_data:
return token_data['key']
return None

2
litellm/litellm_core_utils/custom_logger_registry.py
View file

@ -47,6 +47,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i
VectorStorePreCallHook,
)
from litellm.proxy.hooks.dynamic_rate_limiter import _PROXY_DynamicRateLimitHandler
from litellm.proxy.hooks.dynamic_rate_limiter_v3 import _PROXY_DynamicRateLimitHandlerV3
class CustomLoggerRegistry:
@ -86,6 +87,7 @@ class CustomLoggerRegistry:
"s3_v2": S3Logger,
"aws_sqs": SQSLogger,
"dynamic_rate_limiter": _PROXY_DynamicRateLimitHandler,
"dynamic_rate_limiter_v3": _PROXY_DynamicRateLimitHandlerV3,
"vector_store_pre_call_hook": VectorStorePreCallHook,
"dotprompt": DotpromptManager,
"cloudzero": CloudZeroLogger,

1
litellm/litellm_core_utils/duration_parser.py
View file

@ -158,6 +158,7 @@ def _setup_timezone(
"US/Eastern": timezone(timedelta(hours=-4)), # EDT
"US/Pacific": timezone(timedelta(hours=-7)), # PDT
"Asia/Kolkata": timezone(timedelta(hours=5, minutes=30)), # IST
"Asia/Bangkok": timezone(timedelta(hours=7)), # ICT (Indochina Time)
"Europe/London": timezone(timedelta(hours=1)), # BST
"UTC": timezone.utc,
}

160
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -6,6 +6,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.types.utils import LlmProviders
from ..exceptions import (
APIConnectionError,
@ -762,7 +763,7 @@ def exception_type( # type: ignore # noqa: PLR0915
error_str += "XXXXXXX" + '"'
raise AuthenticationError(
message=f"{custom_llm_provider}Exception: Authentication Error - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception: Authentication Error - {error_str}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
@ -771,14 +772,14 @@ def exception_type( # type: ignore # noqa: PLR0915
elif "model's maximum context limit" in error_str:
exception_mapping_worked = True
raise ContextWindowExceededError(
message=f"{custom_llm_provider}Exception: Context Window Error - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception: Context Window Error - {error_str}",
model=model,
llm_provider=custom_llm_provider,
)
elif "token_quota_reached" in error_str:
exception_mapping_worked = True
raise RateLimitError(
message=f"{custom_llm_provider}Exception: Rate Limit Errror - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception: Rate Limit Errror - {error_str}",
llm_provider=custom_llm_provider,
model=model,
response=getattr(original_exception, "response", None),
@ -789,14 +790,14 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif "model_no_support_for_function" in error_str:
exception_mapping_worked = True
raise BadRequestError(
message=f"{custom_llm_provider}Exception - Use 'watsonx_text' route instead. IBM WatsonX does not support `/text/chat` endpoint. - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception - Use 'watsonx_text' route instead. IBM WatsonX does not support `/text/chat` endpoint. - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
@ -804,7 +805,7 @@ def exception_type( # type: ignore # noqa: PLR0915
if original_exception.status_code == 500:
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
@ -814,28 +815,28 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise AuthenticationError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 400:
exception_mapping_worked = True
raise BadRequestError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 404:
exception_mapping_worked = True
raise NotFoundError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 408:
exception_mapping_worked = True
raise Timeout(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@ -846,7 +847,7 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise BadRequestError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@ -854,7 +855,7 @@ def exception_type( # type: ignore # noqa: PLR0915
elif original_exception.status_code == 429:
exception_mapping_worked = True
raise RateLimitError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@ -862,7 +863,7 @@ def exception_type( # type: ignore # noqa: PLR0915
elif original_exception.status_code == 503:
exception_mapping_worked = True
raise ServiceUnavailableError(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@ -870,7 +871,7 @@ def exception_type( # type: ignore # noqa: PLR0915
elif original_exception.status_code == 504: # gateway timeout error
exception_mapping_worked = True
raise Timeout(
message=f"{custom_llm_provider}Exception - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@ -1168,9 +1169,9 @@ def exception_type( # type: ignore # noqa: PLR0915
exception_status_code=original_exception.status_code,
)
elif (
custom_llm_provider == "vertex_ai"
or custom_llm_provider == "vertex_ai_beta"
or custom_llm_provider == "gemini"
custom_llm_provider == LlmProviders.VERTEX_AI
or custom_llm_provider == LlmProviders.VERTEX_AI_BETA
or custom_llm_provider == LlmProviders.GEMINI
):
if (
"Vertex AI API has not been used in project" in error_str
@ -1178,9 +1179,9 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise BadRequestError(
message=f"litellm.BadRequestError: VertexAIException - {error_str}",
message=f"litellm.BadRequestError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=400,
request=httpx.Request(
@ -1193,7 +1194,7 @@ def exception_type( # type: ignore # noqa: PLR0915
if "400 Request payload size exceeds" in error_str:
exception_mapping_worked = True
raise ContextWindowExceededError(
message=f"VertexException - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
model=model,
llm_provider=custom_llm_provider,
)
@ -1203,9 +1204,9 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"litellm.InternalServerError: VertexAIException - {error_str}",
message=f"litellm.InternalServerError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=500,
content=str(original_exception),
@ -1216,7 +1217,7 @@ def exception_type( # type: ignore # noqa: PLR0915
elif "API key not valid." in error_str:
exception_mapping_worked = True
raise AuthenticationError(
message=f"{custom_llm_provider}Exception - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
@ -1224,9 +1225,9 @@ def exception_type( # type: ignore # noqa: PLR0915
elif "403" in error_str:
exception_mapping_worked = True
raise BadRequestError(
message=f"VertexAIException BadRequestError - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
response=httpx.Response(
status_code=403,
request=httpx.Request(
@ -1243,9 +1244,9 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise ContentPolicyViolationError(
message=f"VertexAIException ContentPolicyViolationError - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception ContentPolicyViolationError - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=400,
@ -1264,9 +1265,9 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise RateLimitError(
message=f"litellm.RateLimitError: VertexAIException - {error_str}",
message=f"litellm.RateLimitError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=429,
@ -1282,18 +1283,18 @@ def exception_type( # type: ignore # noqa: PLR0915
):
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"litellm.InternalServerError: VertexAIException - {error_str}",
message=f"litellm.InternalServerError: {custom_llm_provider}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
)
if hasattr(original_exception, "status_code"):
if original_exception.status_code == 400:
exception_mapping_worked = True
raise BadRequestError(
message=f"VertexAIException BadRequestError - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=400,
@ -1306,21 +1307,35 @@ def exception_type( # type: ignore # noqa: PLR0915
if original_exception.status_code == 401:
exception_mapping_worked = True
raise AuthenticationError(
message=f"VertexAIException - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
if original_exception.status_code == 403:
exception_mapping_worked = True
raise PermissionDeniedError(
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
response=httpx.Response(
status_code=403,
request=httpx.Request(
method="POST",
url="https://cloud.google.com/vertex-ai/",
),
),
)
if original_exception.status_code == 404:
exception_mapping_worked = True
raise NotFoundError(
message=f"VertexAIException - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
if original_exception.status_code == 408:
exception_mapping_worked = True
raise Timeout(
message=f"VertexAIException - {original_exception.message}",
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
@ -1328,9 +1343,9 @@ def exception_type( # type: ignore # noqa: PLR0915
if original_exception.status_code == 429:
exception_mapping_worked = True
raise RateLimitError(
message=f"litellm.RateLimitError: VertexAIException - {error_str}",
message=f"litellm.RateLimitError: {custom_llm_provider.capitalize()}Exception - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=429,
@ -1343,9 +1358,9 @@ def exception_type( # type: ignore # noqa: PLR0915
if original_exception.status_code == 500:
exception_mapping_worked = True
raise litellm.InternalServerError(
message=f"VertexAIException InternalServerError - {error_str}",
message=f"{custom_llm_provider.capitalize()}Exception InternalServerError - {error_str}",
model=model,
llm_provider="vertex_ai",
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
response=httpx.Response(
status_code=500,
@ -1353,71 +1368,20 @@ def exception_type( # type: ignore # noqa: PLR0915
request=httpx.Request(method="completion", url="https://github.com/BerriAI/litellm"), # type: ignore
),
)
if original_exception.status_code == 503:
if original_exception.status_code == 502:
exception_mapping_worked = True
raise ServiceUnavailableError(
message=f"VertexAIException - {original_exception.message}",
raise APIConnectionError(
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
)
elif custom_llm_provider == "palm" or custom_llm_provider == "gemini":
if "503 Getting metadata" in error_str:
# auth errors look like this
# 503 Getting metadata from plugin failed with error: Reauthentication is needed. Please run `gcloud auth application-default login` to reauthenticate.
exception_mapping_worked = True
raise BadRequestError(
message="GeminiException - Invalid api key",
model=model,
llm_provider="palm",
response=getattr(original_exception, "response", None),
)
if (
"504 Deadline expired before operation could complete." in error_str
or "504 Deadline Exceeded" in error_str
):
exception_mapping_worked = True
raise Timeout(
message=f"GeminiException - {original_exception.message}",
model=model,
llm_provider="palm",
exception_status_code=original_exception.status_code,
)
if "400 Request payload size exceeds" in error_str:
exception_mapping_worked = True
raise ContextWindowExceededError(
message=f"GeminiException - {error_str}",
model=model,
llm_provider="palm",
response=getattr(original_exception, "response", None),
)
if (
"500 An internal error has occurred." in error_str
or "list index out of range" in error_str
):
exception_mapping_worked = True
raise APIError(
status_code=getattr(original_exception, "status_code", 500),
message=f"GeminiException - {original_exception.message}",
llm_provider="palm",
model=model,
request=httpx.Response(
status_code=429,
request=httpx.Request(
method="POST",
url=" https://cloud.google.com/vertex-ai/",
),
),
)
if hasattr(original_exception, "status_code"):
if original_exception.status_code == 400:
if original_exception.status_code == 503:
exception_mapping_worked = True
raise BadRequestError(
message=f"GeminiException - {error_str}",
raise ServiceUnavailableError(
message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
llm_provider=custom_llm_provider,
model=model,
llm_provider="palm",
response=getattr(original_exception, "response", None),
)
# Dailed: Error occurred: 400 Request payload size exceeds the limit: 20000 bytes
elif custom_llm_provider == "cloudflare":
if "Authentication error" in error_str:
exception_mapping_worked = True

10
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -252,6 +252,9 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "https://ai-gateway.vercel.sh/v1":
custom_llm_provider = "vercel_ai_gateway"
dynamic_api_key = get_secret_str("VERCEL_AI_GATEWAY_API_KEY")
elif endpoint == "https://api.inference.wandb.ai/v1":
custom_llm_provider = "wandb"
dynamic_api_key = get_secret_str("WANDB_API_KEY")
if api_base is not None and not isinstance(api_base, str):
raise Exception(
@ -773,6 +776,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915
) = litellm.AIMLChatConfig()._get_openai_compatible_provider_info(
api_base, api_key
)
elif custom_llm_provider == "wandb":
api_base = (
api_base
or get_secret("WANDB_API_BASE")
or "https://api.inference.wandb.ai/v1"
) # type: ignore
dynamic_api_key = api_key or get_secret_str("WANDB_API_KEY")
if api_base is not None and not isinstance(api_base, str):
raise Exception("api base needs to be a string. api_base={}".format(api_base))

3
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -149,6 +149,9 @@ def get_supported_openai_params( # noqa: PLR0915
elif custom_llm_provider == "nebius":
if request_type == "chat_completion":
return litellm.NebiusConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "wandb":
if request_type == "chat_completion":
return litellm.WandbConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "replicate":
return litellm.ReplicateConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "huggingface":

35
litellm/litellm_core_utils/litellm_logging.py
View file

@ -26,7 +26,6 @@ from typing import (
cast,
)
import fastuuid as uuid
from httpx import Response
from pydantic import BaseModel
@ -38,6 +37,7 @@ from litellm import (
turn_off_message_logging,
)
from litellm._logging import _is_debugging_on, verbose_logger
from litellm._uuid import uuid
from litellm.batches.batch_utils import _handle_completed_batch
from litellm.caching.caching import DualCache, InMemoryCache
from litellm.caching.caching_handler import LLMCachingHandler
@ -1228,6 +1228,7 @@ class Logging(LiteLLMLoggingBaseClass):
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,
"litellm_logging_obj": self,
"service_tier": self.optional_params.get("service_tier") if self.optional_params else None,
}
except Exception as e: # error creating kwargs for cost calculation
debug_info = StandardLoggingModelCostFailureDebugInformation(
@ -3444,6 +3445,30 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
dynamic_rate_limiter_obj.update_variables(llm_router=llm_router)
_in_memory_loggers.append(dynamic_rate_limiter_obj)
return dynamic_rate_limiter_obj # type: ignore
elif logging_integration == "dynamic_rate_limiter_v3":
from litellm.proxy.hooks.dynamic_rate_limiter_v3 import (
_PROXY_DynamicRateLimitHandlerV3,
)
for callback in _in_memory_loggers:
if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3):
return callback # type: ignore
if internal_usage_cache is None:
raise Exception(
"Internal Error: Cache cannot be empty - internal_usage_cache={}".format(
internal_usage_cache
)
)
dynamic_rate_limiter_obj_v3 = _PROXY_DynamicRateLimitHandlerV3(
internal_usage_cache=internal_usage_cache
)
if llm_router is not None and isinstance(llm_router, litellm.Router):
dynamic_rate_limiter_obj_v3.update_variables(llm_router=llm_router)
_in_memory_loggers.append(dynamic_rate_limiter_obj_v3)
return dynamic_rate_limiter_obj_v3 # type: ignore
elif logging_integration == "langtrace":
if "LANGTRACE_API_KEY" not in os.environ:
raise ValueError("LANGTRACE_API_KEY not found in environment variables")
@ -3707,6 +3732,14 @@ def get_custom_logger_compatible_class( # noqa: PLR0915
for callback in _in_memory_loggers:
if isinstance(callback, _PROXY_DynamicRateLimitHandler):
return callback # type: ignore
elif logging_integration == "dynamic_rate_limiter_v3":
from litellm.proxy.hooks.dynamic_rate_limiter_v3 import (
_PROXY_DynamicRateLimitHandlerV3,
)
for callback in _in_memory_loggers:
if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3):
return callback # type: ignore
elif logging_integration == "langtrace":
from litellm.integrations.opentelemetry import OpenTelemetry

408
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -1,16 +1,18 @@
# What is this?
## Helper utilities for cost_per_token()
from typing import Any, Literal, Optional, Tuple, cast
from typing import Any, Literal, Optional, Tuple, TypedDict, cast
import litellm
from litellm._logging import verbose_logger
from litellm.types.utils import (
CacheCreationTokenDetails,
CallTypes,
ImageResponse,
ModelInfo,
PassthroughCallTypes,
Usage,
ServiceTier,
)
from litellm.utils import get_model_info
@ -113,9 +115,31 @@ def _generic_cost_per_character(
return prompt_cost, completion_cost
def _get_service_tier_cost_key(base_key: str, service_tier: Optional[str]) -> str:
"""
Get the appropriate cost key based on service tier.
Args:
base_key: The base cost key (e.g., "input_cost_per_token")
service_tier: The service tier ("flex", "priority", or None for standard)
Returns:
str: The cost key to use (e.g., "input_cost_per_token_flex" or "input_cost_per_token")
"""
if service_tier is None:
return base_key
# Only use service tier specific keys for "flex" and "priority"
if service_tier.lower() in [ServiceTier.FLEX.value, ServiceTier.PRIORITY.value]:
return f"{base_key}_{service_tier.lower()}"
# For any other service tier, use standard pricing
return base_key
def _get_token_base_cost(
model_info: ModelInfo, usage: Usage
) -> Tuple[float, float, float, float]:
model_info: ModelInfo, usage: Usage, service_tier: Optional[str] = None
) -> Tuple[float, float, float, float, float]:
"""
Return prompt cost, completion cost, and cache costs for a given model and usage.
@ -125,17 +149,27 @@ def _get_token_base_cost(
Returns:
Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
"""
# Get service tier aware cost keys
input_cost_key = _get_service_tier_cost_key("input_cost_per_token", service_tier)
output_cost_key = _get_service_tier_cost_key("output_cost_per_token", service_tier)
cache_creation_cost_key = _get_service_tier_cost_key("cache_creation_input_token_cost", service_tier)
cache_read_cost_key = _get_service_tier_cost_key("cache_read_input_token_cost", service_tier)
prompt_base_cost = cast(
float, _get_cost_per_unit(model_info, "input_cost_per_token")
float, _get_cost_per_unit(model_info, input_cost_key)
)
completion_base_cost = cast(
float, _get_cost_per_unit(model_info, "output_cost_per_token")
float, _get_cost_per_unit(model_info, output_cost_key)
)
cache_creation_cost = cast(
float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost")
float, _get_cost_per_unit(model_info, cache_creation_cost_key)
)
cache_creation_cost_above_1hr = cast(
float,
_get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"),
)
cache_read_cost = cast(
float, _get_cost_per_unit(model_info, "cache_read_input_token_cost")
float, _get_cost_per_unit(model_info, cache_read_cost_key)
)
## CHECK IF ABOVE THRESHOLD
@ -194,7 +228,13 @@ def _get_token_base_cost(
except Exception:
continue
return prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost
return (
prompt_base_cost,
completion_base_cost,
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
)
def calculate_cost_component(
@ -238,11 +278,224 @@ def _get_cost_per_unit(
verbose_logger.exception(
f"litellm.litellm_core_utils.llm_cost_calc.utils.py::calculate_cost_per_component(): Exception occured - {cost_per_unit}\nDefaulting to 0.0"
)
# If the service tier key doesn't exist or is None, try to fall back to the standard key
if cost_per_unit is None:
# Check if any service tier suffix exists in the cost key using ServiceTier enum
for service_tier in ServiceTier:
suffix = f"_{service_tier.value}"
if suffix in cost_key:
# Extract the base key by removing the matched suffix
base_key = cost_key.replace(suffix, '')
fallback_cost = model_info.get(base_key)
if isinstance(fallback_cost, float):
return fallback_cost
if isinstance(fallback_cost, int):
return float(fallback_cost)
if isinstance(fallback_cost, str):
try:
return float(fallback_cost)
except ValueError:
verbose_logger.exception(
f"litellm.litellm_core_utils.llm_cost_calc.utils.py::_get_cost_per_unit(): Exception occured - {fallback_cost}\nDefaulting to 0.0"
)
break # Only try the first matching suffix
return default_value
def calculate_cache_writing_cost(
cache_creation_tokens: int,
cache_creation_token_details: Optional[CacheCreationTokenDetails],
cache_creation_cost_above_1hr: float,
cache_creation_cost: float,
) -> float:
"""
Adjust cost of cache creation tokens based on the cache creation token details.
"""
total_cost: float = 0.0
if cache_creation_token_details is not None:
# get the number of 5m and 1h cache creation tokens
cache_creation_tokens_5m = (
cache_creation_token_details.ephemeral_5m_input_tokens
)
cache_creation_tokens_1h = (
cache_creation_token_details.ephemeral_1h_input_tokens
)
# add the number of 5m and 1h cache creation tokens to the cache creation tokens
total_cost += (
cache_creation_tokens_5m * cache_creation_cost
if cache_creation_tokens_5m is not None
else 0.0
)
total_cost += (
cache_creation_tokens_1h * cache_creation_cost_above_1hr
if cache_creation_tokens_1h is not None
else 0.0
)
else:
total_cost += cache_creation_tokens * cache_creation_cost
return total_cost
class PromptTokensDetailsResult(TypedDict):
cache_hit_tokens: int
cache_creation_tokens: int
cache_creation_token_details: Optional[CacheCreationTokenDetails]
text_tokens: int
audio_tokens: int
character_count: int
image_count: int
video_length_seconds: int
def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
cache_hit_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "cached_tokens", 0))
or 0
)
cache_creation_tokens = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "cache_creation_tokens", 0),
)
or 0
)
cache_creation_token_details = (
cast(
Optional[CacheCreationTokenDetails],
getattr(usage.prompt_tokens_details, "cache_creation_token_details", None),
)
or None
)
text_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "text_tokens", None))
or 0 # default to prompt tokens, if this field is not set
)
audio_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0))
or 0
)
character_count = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "character_count", 0),
)
or 0
)
image_count = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "image_count", 0)) or 0
)
video_length_seconds = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "video_length_seconds", 0),
)
or 0
)
return PromptTokensDetailsResult(
cache_hit_tokens=cache_hit_tokens,
cache_creation_tokens=cache_creation_tokens,
cache_creation_token_details=cache_creation_token_details,
text_tokens=text_tokens,
audio_tokens=audio_tokens,
character_count=character_count,
image_count=image_count,
video_length_seconds=video_length_seconds,
)
class CompletionTokensDetailsResult(TypedDict):
audio_tokens: int
text_tokens: int
reasoning_tokens: int
def _parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResult:
audio_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "audio_tokens", 0),
)
or 0
)
text_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "text_tokens", None),
)
or 0 # default to completion tokens, if this field is not set
)
reasoning_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "reasoning_tokens", 0),
)
or 0
)
return CompletionTokensDetailsResult(
audio_tokens=audio_tokens,
text_tokens=text_tokens,
reasoning_tokens=reasoning_tokens,
)
def _calculate_input_cost(
prompt_tokens_details: PromptTokensDetailsResult,
model_info: ModelInfo,
prompt_base_cost: float,
cache_read_cost: float,
cache_creation_cost: float,
cache_creation_cost_above_1hr: float,
) -> float:
"""
Calculates the input cost for a given model, prompt tokens, and completion tokens.
"""
prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
### AUDIO COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_audio_token", prompt_tokens_details["audio_tokens"]
)
### CACHE WRITING COST - Now uses tiered pricing
prompt_cost += calculate_cache_writing_cost(
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
cache_creation_token_details=prompt_tokens_details[
"cache_creation_token_details"
],
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
cache_creation_cost=cache_creation_cost,
)
### CHARACTER COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_character", prompt_tokens_details["character_count"]
)
### IMAGE COUNT COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_image", prompt_tokens_details["image_count"]
)
### VIDEO LENGTH COST
prompt_cost += calculate_cost_component(
model_info,
"input_cost_per_video_per_second",
prompt_tokens_details["video_length_seconds"],
)
return prompt_cost
def generic_cost_per_token(
model: str, usage: Usage, custom_llm_provider: str
model: str, usage: Usage, custom_llm_provider: str, service_tier: Optional[str] = None
) -> Tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@ -264,97 +517,45 @@ def generic_cost_per_token(
### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing)
prompt_cost = 0.0
### PROCESSING COST
text_tokens = usage.prompt_tokens
cache_hit_tokens = 0
cache_creation_tokens = 0
audio_tokens = 0
character_count = 0
image_count = 0
video_length_seconds = 0
prompt_tokens_details = PromptTokensDetailsResult(
cache_hit_tokens=0,
cache_creation_tokens=0,
cache_creation_token_details=None,
text_tokens=usage.prompt_tokens,
audio_tokens=0,
character_count=0,
image_count=0,
video_length_seconds=0,
)
if usage.prompt_tokens_details:
cache_hit_tokens = (
cast(
Optional[int], getattr(usage.prompt_tokens_details, "cached_tokens", 0)
)
or 0
)
cache_creation_tokens = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "cache_creation_tokens", 0),
)
or 0
)
text_tokens = (
cast(
Optional[int], getattr(usage.prompt_tokens_details, "text_tokens", None)
)
or 0 # default to prompt tokens, if this field is not set
)
audio_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0))
or 0
)
character_count = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "character_count", 0),
)
or 0
)
image_count = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "image_count", 0))
or 0
)
video_length_seconds = (
cast(
Optional[int],
getattr(usage.prompt_tokens_details, "video_length_seconds", 0),
)
or 0
)
prompt_tokens_details = _parse_prompt_tokens_details(usage)
## EDGE CASE - text tokens not set inside PromptTokensDetails
if text_tokens == 0:
if prompt_tokens_details["text_tokens"] == 0:
text_tokens = (
usage.prompt_tokens
- cache_hit_tokens
- audio_tokens
- cache_creation_tokens
- prompt_tokens_details["cache_hit_tokens"]
- prompt_tokens_details["audio_tokens"]
- prompt_tokens_details["cache_creation_tokens"]
)
prompt_tokens_details["text_tokens"] = text_tokens
prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost = (
_get_token_base_cost(model_info=model_info, usage=usage)
)
(
prompt_base_cost,
completion_base_cost,
cache_creation_cost,
cache_creation_cost_above_1hr,
cache_read_cost,
) = _get_token_base_cost(model_info=model_info, usage=usage, service_tier=service_tier)
prompt_cost = float(text_tokens) * prompt_base_cost
### CACHE READ COST - Now uses tiered pricing
prompt_cost += float(cache_hit_tokens) * cache_read_cost
### AUDIO COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_audio_token", audio_tokens
)
### CACHE WRITING COST - Now uses tiered pricing
prompt_cost += float(cache_creation_tokens) * cache_creation_cost
### CHARACTER COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_character", character_count
)
### IMAGE COUNT COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_image", image_count
)
### VIDEO LENGTH COST
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_video_per_second", video_length_seconds
prompt_cost = _calculate_input_cost(
prompt_tokens_details=prompt_tokens_details,
model_info=model_info,
prompt_base_cost=prompt_base_cost,
cache_read_cost=cache_read_cost,
cache_creation_cost=cache_creation_cost,
cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
)
## CALCULATE OUTPUT COST
@ -363,27 +564,10 @@ def generic_cost_per_token(
reasoning_tokens = 0
is_text_tokens_total = False
if usage.completion_tokens_details is not None:
audio_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "audio_tokens", 0),
)
or 0
)
text_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "text_tokens", None),
)
or 0 # default to completion tokens, if this field is not set
)
reasoning_tokens = (
cast(
Optional[int],
getattr(usage.completion_tokens_details, "reasoning_tokens", 0),
)
or 0
)
completion_tokens_details = _parse_completion_tokens_details(usage)
audio_tokens = completion_tokens_details["audio_tokens"]
text_tokens = completion_tokens_details["text_tokens"]
reasoning_tokens = completion_tokens_details["reasoning_tokens"]
if text_tokens == 0:
text_tokens = usage.completion_tokens

7
litellm/litellm_core_utils/object_pooling.py
View file

@ -14,8 +14,9 @@ Memory Management Strategy:
- Unlimited pools when maxsize is not specified (eviction controls actual usage)
"""
from typing import Type, TypeVar, Optional, Callable
from pond import Pond, PooledObjectFactory, PooledObject
from typing import Any, Callable, Optional, Type, TypeVar
from pond import Pond, PooledObject, PooledObjectFactory
T = TypeVar('T')
@ -50,7 +51,7 @@ class GenericPooledObjectFactory(PooledObjectFactory):
pooled_object.keeped_object.__dict__.clear()
del pooled_object
def reset(self, pooled_object: PooledObject) -> PooledObject:
def reset(self, pooled_object: PooledObject, **kwargs: Any) -> PooledObject:
"""Reset the pooled object to a clean state."""
obj = pooled_object.keeped_object
# Reset the object by calling its reset method if it exists

11
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1619,11 +1619,12 @@ class CustomStreamWrapper:
completion_start_time=datetime.datetime.now()
)
## LOGGING
executor.submit(
self.run_success_logging_and_cache_storage,
response,
cache_hit,
) # log response
if not litellm.disable_streaming_logging:
executor.submit(
self.run_success_logging_and_cache_storage,
response,
cache_hit,
) # log response
choice = response.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""

28
litellm/llms/anthropic/chat/transformation.py
View file

@ -45,7 +45,10 @@ from litellm.types.llms.openai import (
OpenAIMcpServerTool,
OpenAIWebSearchOptions,
)
from litellm.types.utils import CompletionTokensDetailsWrapper
from litellm.types.utils import (
CacheCreationTokenDetails,
CompletionTokensDetailsWrapper,
)
from litellm.types.utils import Message as LitellmMessage
from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse
from litellm.utils import (
@ -801,7 +804,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if content.get("citations") is not None:
if citations is None:
citations = []
citations.append(content["citations"])
citations.append(
[
{
**citation,
"supported_text": content.get("text", ""),
}
for citation in content["citations"]
]
)
if thinking_blocks is not None:
reasoning_content = ""
for block in thinking_blocks:
@ -820,6 +831,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_usage = usage_object
cache_creation_input_tokens: int = 0
cache_read_input_tokens: int = 0
cache_creation_token_details: Optional[CacheCreationTokenDetails] = None
web_search_requests: Optional[int] = None
if (
"cache_creation_input_tokens" in _usage
@ -842,8 +854,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
int, _usage["server_tool_use"]["web_search_requests"]
)
if "cache_creation" in _usage and _usage["cache_creation"] is not None:
cache_creation_token_details = CacheCreationTokenDetails(
ephemeral_5m_input_tokens=_usage["cache_creation"].get(
"ephemeral_5m_input_tokens"
),
ephemeral_1h_input_tokens=_usage["cache_creation"].get(
"ephemeral_1h_input_tokens"
),
)
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=cache_read_input_tokens,
cache_creation_tokens=cache_read_input_tokens,
cache_creation_token_details=cache_creation_token_details,
)
completion_token_details = (
CompletionTokensDetailsWrapper(

7
litellm/llms/azure/chat/o_series_handler.py
View file

@ -4,7 +4,7 @@ Handler file for calls to Azure OpenAI's o1/o3 family of models
Written separately to handle faking streaming for o1 and o3 models.
"""
from typing import Any, Callable, Optional, Union
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
import httpx
@ -13,6 +13,9 @@ from litellm.types.utils import ModelResponse
from ...openai.openai import OpenAIChatCompletion
from ..common_utils import BaseAzureLLM
if TYPE_CHECKING:
from aiohttp import ClientSession
class AzureOpenAIO1ChatCompletion(BaseAzureLLM, OpenAIChatCompletion):
def completion(
@ -38,6 +41,7 @@ class AzureOpenAIO1ChatCompletion(BaseAzureLLM, OpenAIChatCompletion):
organization: Optional[str] = None,
custom_llm_provider: Optional[str] = None,
drop_params: Optional[bool] = None,
shared_session: Optional["ClientSession"] = None,
):
client = self.get_azure_openai_client(
litellm_params=litellm_params,
@ -69,4 +73,5 @@ class AzureOpenAIO1ChatCompletion(BaseAzureLLM, OpenAIChatCompletion):
organization=organization,
custom_llm_provider=custom_llm_provider,
drop_params=drop_params,
shared_session=shared_session,
)

15
litellm/llms/azure_ai/image_edit/__init__.py Normal file
View file

@ -0,0 +1,15 @@
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from .transformation import AzureFoundryFluxImageEditConfig
__all__ = ["AzureFoundryFluxImageEditConfig"]
def get_azure_ai_image_edit_config(model: str) -> BaseImageEditConfig:
model = model.lower()
model = model.replace("-", "")
model = model.replace("_", "")
if model == "" or "flux" in model: # empty model is flux
return AzureFoundryFluxImageEditConfig()
else:
raise ValueError(f"Model {model} is not supported for Azure AI image editing.")

99
litellm/llms/azure_ai/image_edit/transformation.py Normal file
View file

@ -0,0 +1,99 @@
from typing import Optional
import httpx
import litellm
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
from litellm.secret_managers.main import get_secret_str
from litellm.utils import _add_path_to_api_base
class AzureFoundryFluxImageEditConfig(OpenAIImageEditConfig):
"""
Azure AI Foundry FLUX image edit config
Supports FLUX models including FLUX-1-kontext-pro for image editing.
Azure AI Foundry FLUX models handle image editing through the /images/edits endpoint,
same as standard Azure OpenAI models. The request format uses multipart/form-data
with image files and prompt.
"""
def validate_environment(
self,
headers: dict,
model: str,
api_key: Optional[str] = None,
) -> dict:
"""
Validate Azure AI Foundry environment and set up authentication
Uses Api-Key header format
"""
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if not api_key:
raise ValueError(
f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
)
headers.update(
{
"Api-Key": api_key, # Azure AI Foundry uses Api-Key header format
}
)
return headers
def get_complete_url(
self,
model: str,
api_base: Optional[str],
litellm_params: dict,
) -> str:
"""
Constructs a complete URL for Azure AI Foundry image edits API request.
Azure AI Foundry FLUX models handle image editing through the /images/edits
endpoint.
Args:
- model: Model name (deployment name for Azure AI Foundry)
- api_base: Base URL for Azure AI endpoint
- litellm_params: Additional parameters including api_version
Returns:
- Complete URL for the image edits endpoint
"""
api_base = AzureFoundryModelInfo.get_api_base(api_base)
if api_base is None:
raise ValueError(
"Azure AI API base is required. Set AZURE_AI_API_BASE environment variable or pass api_base parameter."
)
api_version = (litellm_params.get("api_version") or litellm.api_version
or get_secret_str("AZURE_AI_API_VERSION")
)
if api_version is None:
# API version is mandatory for Azure AI Foundry
raise ValueError(
"Azure API version is required. Set AZURE_AI_API_VERSION environment variable or pass api_version parameter."
)
# Add the path to the base URL using the model as deployment name
# Azure AI Foundry FLUX models use /images/edits for editing
if "/openai/deployments/" in api_base:
new_url = _add_path_to_api_base(
api_base=api_base,
ending_path="/images/edits",
)
else:
new_url = _add_path_to_api_base(
api_base=api_base,
ending_path=f"/openai/deployments/{model}/images/edits",
)
# Use the new query_params dictionary
final_url = httpx.URL(new_url).copy_with(params={"api-version": api_version})
return str(final_url)

198
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -175,6 +175,77 @@ class AmazonConverseConfig(BaseConfig):
and v is not None
}
def _validate_request_metadata(self, metadata: dict) -> None:
"""
Validate requestMetadata according to AWS Bedrock Converse API constraints.
Constraints:
- Maximum of 16 items
- Keys: 1-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{1,256}
- Values: 0-256 characters, pattern [a-zA-Z0-9\\s:_@$#=/+,-.]{0,256}
"""
import re
if not isinstance(metadata, dict):
raise litellm.exceptions.BadRequestError(
message="requestMetadata must be a dictionary",
model="bedrock",
llm_provider="bedrock",
)
if len(metadata) > 16:
raise litellm.exceptions.BadRequestError(
message="requestMetadata can contain a maximum of 16 items",
model="bedrock",
llm_provider="bedrock",
)
key_pattern = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{1,256}$")
value_pattern = re.compile(r"^[a-zA-Z0-9\s:_@$#=/+,.-]{0,256}$")
for key, value in metadata.items():
if not isinstance(key, str):
raise litellm.exceptions.BadRequestError(
message="requestMetadata keys must be strings",
model="bedrock",
llm_provider="bedrock",
)
if not isinstance(value, str):
raise litellm.exceptions.BadRequestError(
message="requestMetadata values must be strings",
model="bedrock",
llm_provider="bedrock",
)
if len(key) == 0 or len(key) > 256:
raise litellm.exceptions.BadRequestError(
message="requestMetadata key length must be 1-256 characters",
model="bedrock",
llm_provider="bedrock",
)
if len(value) > 256:
raise litellm.exceptions.BadRequestError(
message="requestMetadata value length must be 0-256 characters",
model="bedrock",
llm_provider="bedrock",
)
if not key_pattern.match(key):
raise litellm.exceptions.BadRequestError(
message=f"requestMetadata key '{key}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
model="bedrock",
llm_provider="bedrock",
)
if not value_pattern.match(value):
raise litellm.exceptions.BadRequestError(
message=f"requestMetadata value '{value}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]",
model="bedrock",
llm_provider="bedrock",
)
def get_supported_openai_params(self, model: str) -> List[str]:
from litellm.utils import supports_function_calling
@ -188,6 +259,7 @@ class AmazonConverseConfig(BaseConfig):
"top_p",
"extra_headers",
"response_format",
"requestMetadata",
]
if (
@ -497,6 +569,10 @@ class AmazonConverseConfig(BaseConfig):
optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
value
)
if param == "requestMetadata":
if value is not None and isinstance(value, dict):
self._validate_request_metadata(value) # type: ignore
optional_params["requestMetadata"] = value
# Only update thinking tokens for non-GPT-OSS models
if "gpt-oss" not in model:
@ -686,34 +762,10 @@ class AmazonConverseConfig(BaseConfig):
return {}
def _transform_request_helper(
self,
model: str,
system_content_blocks: List[SystemContentBlock],
optional_params: dict,
messages: Optional[List[AllMessageValues]] = None,
headers: Optional[dict] = None,
) -> CommonRequestObject:
## VALIDATE REQUEST
"""
Bedrock doesn't support tool calling without `tools=` param specified.
"""
if (
"tools" not in optional_params
and messages is not None
and has_tool_call_blocks(messages)
):
if litellm.modify_params:
optional_params["tools"] = add_dummy_tool(
custom_llm_provider="bedrock_converse"
)
else:
raise litellm.UnsupportedParamsError(
message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.",
model="",
llm_provider="bedrock",
)
def _prepare_request_params(
self, optional_params: dict, model: str
) -> Tuple[dict, dict, dict]:
"""Prepare and separate request parameters."""
inference_params = copy.deepcopy(optional_params)
supported_converse_params = list(
AmazonConverseConfig.__annotations__.keys()
@ -727,6 +779,11 @@ class AmazonConverseConfig(BaseConfig):
)
inference_params.pop("json_mode", None) # used for handling json_schema
# Extract requestMetadata before processing other parameters
request_metadata = inference_params.pop("requestMetadata", None)
if request_metadata is not None:
self._validate_request_metadata(request_metadata)
# keep supported params in 'inference_params', and set all model-specific params in 'additional_request_params'
additional_request_params = {
k: v for k, v in inference_params.items() if k not in total_supported_params
@ -740,9 +797,16 @@ class AmazonConverseConfig(BaseConfig):
self._handle_top_k_value(model, inference_params)
)
original_tools = inference_params.pop("tools", [])
return inference_params, additional_request_params, request_metadata
# Initialize bedrock_tools
def _process_tools_and_beta(
self,
original_tools: list,
model: str,
headers: Optional[dict],
additional_request_params: dict,
) -> Tuple[List[ToolBlock], list]:
"""Process tools and collect anthropic_beta values."""
bedrock_tools: List[ToolBlock] = []
# Collect anthropic_beta values from user headers
@ -784,6 +848,48 @@ class AmazonConverseConfig(BaseConfig):
seen.add(beta)
additional_request_params["anthropic_beta"] = unique_betas
return bedrock_tools, anthropic_beta_list
def _transform_request_helper(
self,
model: str,
system_content_blocks: List[SystemContentBlock],
optional_params: dict,
messages: Optional[List[AllMessageValues]] = None,
headers: Optional[dict] = None,
) -> CommonRequestObject:
## VALIDATE REQUEST
"""
Bedrock doesn't support tool calling without `tools=` param specified.
"""
if (
"tools" not in optional_params
and messages is not None
and has_tool_call_blocks(messages)
):
if litellm.modify_params:
optional_params["tools"] = add_dummy_tool(
custom_llm_provider="bedrock_converse"
)
else:
raise litellm.UnsupportedParamsError(
message="Bedrock doesn't support tool calling without `tools=` param specified. Pass `tools=` param OR set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add dummy tool to the request.",
model="",
llm_provider="bedrock",
)
# Prepare and separate parameters
inference_params, additional_request_params, request_metadata = (
self._prepare_request_params(optional_params, model)
)
original_tools = inference_params.pop("tools", [])
# Process tools and collect beta values
bedrock_tools, anthropic_beta_list = self._process_tools_and_beta(
original_tools, model, headers, additional_request_params
)
bedrock_tool_config: Optional[ToolConfigBlock] = None
if len(bedrock_tools) > 0:
tool_choice_values: ToolChoiceValuesBlock = inference_params.pop(
@ -813,6 +919,10 @@ class AmazonConverseConfig(BaseConfig):
if bedrock_tool_config is not None:
data["toolConfig"] = bedrock_tool_config
# Request Metadata (top-level field)
if request_metadata is not None:
data["requestMetadata"] = request_metadata
return data
async def _async_transform_request(
@ -1059,9 +1169,7 @@ class AmazonConverseConfig(BaseConfig):
return message, returned_finish_reason
def _translate_message_content(
self, content_blocks: List[ContentBlock]
) -> Tuple[
def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[
str,
List[ChatCompletionToolCallChunk],
Optional[List[BedrockConverseReasoningContentBlock]],
@ -1076,9 +1184,9 @@ class AmazonConverseConfig(BaseConfig):
"""
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[
List[BedrockConverseReasoningContentBlock]
] = None
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
None
)
for idx, content in enumerate(content_blocks):
"""
- Content is either a tool response or text
@ -1199,9 +1307,9 @@ class AmazonConverseConfig(BaseConfig):
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[
List[BedrockConverseReasoningContentBlock]
] = None
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
None
)
if message is not None:
(
@ -1214,12 +1322,12 @@ class AmazonConverseConfig(BaseConfig):
chat_completion_message["provider_specific_fields"] = {
"reasoningContentBlocks": reasoningContentBlocks,
}
chat_completion_message[
"reasoning_content"
] = self._transform_reasoning_content(reasoningContentBlocks)
chat_completion_message[
"thinking_blocks"
] = self._transform_thinking_blocks(reasoningContentBlocks)
chat_completion_message["reasoning_content"] = (
self._transform_reasoning_content(reasoningContentBlocks)
)
chat_completion_message["thinking_blocks"] = (
self._transform_thinking_blocks(reasoningContentBlocks)
)
chat_completion_message["content"] = content_str
if (
json_mode is True

2
litellm/llms/bedrock/common_utils.py
View file

@ -774,7 +774,7 @@ class CommonBatchFilesUtils:
Returns:
Unique job name (≤ 63 characters for Bedrock compatibility)
"""
import fastuuid as uuid
from litellm._uuid import uuid
unique_id = str(uuid.uuid4())[:8]
# Format: {prefix}-batch-{model}-{uuid}
# Example: litellm-batch-claude-266c398e

52
litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py
View file

@ -10,7 +10,7 @@ Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-tit
"""
import types
from typing import List, Optional
from typing import List, Optional, Union
from litellm.types.llms.bedrock import (
AmazonTitanV2EmbeddingRequest,
@ -30,9 +30,7 @@ class AmazonTitanV2Config:
normalize: Optional[bool] = None
dimensions: Optional[int] = None
def __init__(
self, normalize: Optional[bool] = None, dimensions: Optional[int] = None
) -> None:
def __init__(self, normalize: Optional[bool] = None, dimensions: Optional[int] = None) -> None:
locals_ = locals().copy()
for key, value in locals_.items():
if key != "self" and value is not None:
@ -57,32 +55,56 @@ class AmazonTitanV2Config:
}
def get_supported_openai_params(self) -> List[str]:
return ["dimensions"]
return ["dimensions", "encoding_format"]
def map_openai_params(
self, non_default_params: dict, optional_params: dict
) -> dict:
def map_openai_params(self, non_default_params: dict, optional_params: dict) -> dict:
for k, v in non_default_params.items():
if k == "dimensions":
optional_params["dimensions"] = v
elif k == "encoding_format":
# Map OpenAI encoding_format to AWS embeddingTypes
if v == "float":
optional_params["embeddingTypes"] = ["float"]
elif v == "base64":
# base64 maps to binary format in AWS
optional_params["embeddingTypes"] = ["binary"]
else:
# For any other encoding format, default to float
optional_params["embeddingTypes"] = ["float"]
return optional_params
def _transform_request(
self, input: str, inference_params: dict
) -> AmazonTitanV2EmbeddingRequest:
def _transform_request(self, input: str, inference_params: dict) -> AmazonTitanV2EmbeddingRequest:
return AmazonTitanV2EmbeddingRequest(inputText=input, **inference_params) # type: ignore
def _transform_response(
self, response_list: List[dict], model: str
) -> EmbeddingResponse:
def _transform_response(self, response_list: List[dict], model: str) -> EmbeddingResponse:
total_prompt_tokens = 0
transformed_responses: List[Embedding] = []
for index, response in enumerate(response_list):
_parsed_response = AmazonTitanV2EmbeddingResponse(**response) # type: ignore
# According to AWS docs, embeddingsByType is always present
# If binary was requested (encoding_format="base64"), use binary data
# Otherwise, use float data from embeddingsByType or fallback to embedding field
embedding_data: Union[List[float], List[int]]
if ("embeddingsByType" in _parsed_response and
"binary" in _parsed_response["embeddingsByType"]):
# Use binary data if available (for encoding_format="base64")
embedding_data = _parsed_response["embeddingsByType"]["binary"]
elif ("embeddingsByType" in _parsed_response and
"float" in _parsed_response["embeddingsByType"]):
# Use float data from embeddingsByType
embedding_data = _parsed_response["embeddingsByType"]["float"]
elif "embedding" in _parsed_response:
# Fallback to legacy embedding field
embedding_data = _parsed_response["embedding"]
else:
raise ValueError(f"No embedding data found in response: {response}")
transformed_responses.append(
Embedding(
embedding=_parsed_response["embedding"],
embedding=embedding_data,
index=index,
object="embedding",
)

9
litellm/llms/bedrock/image/amazon_nova_canvas_transformation.py
View file

@ -7,12 +7,12 @@ from litellm.types.llms.bedrock import (
AmazonNovaCanvasColorGuidedGenerationParams,
AmazonNovaCanvasColorGuidedRequest,
AmazonNovaCanvasImageGenerationConfig,
AmazonNovaCanvasInpaintingParams,
AmazonNovaCanvasInpaintingRequest,
AmazonNovaCanvasRequestBase,
AmazonNovaCanvasTextToImageParams,
AmazonNovaCanvasTextToImageRequest,
AmazonNovaCanvasTextToImageResponse,
AmazonNovaCanvasInpaintingParams,
AmazonNovaCanvasInpaintingRequest,
)
from litellm.types.utils import ImageResponse
@ -67,6 +67,11 @@ class AmazonNovaCanvasConfig:
"""
task_type = optional_params.pop("taskType", "TEXT_IMAGE")
image_generation_config = optional_params.pop("imageGenerationConfig", {})
# Extract model_id parameter to prevent "extraneous key" error from Bedrock API
# Following the same pattern as chat completions and embeddings
unencoded_model_id = optional_params.pop("model_id", None) # noqa: F841
image_generation_config = {**image_generation_config, **optional_params}
if task_type == "TEXT_IMAGE":
text_to_image_params: Dict[str, Any] = image_generation_config.pop(

12
litellm/llms/bedrock/image/image_handler.py
View file

@ -233,7 +233,17 @@ class BedrockImageGeneration(BaseAWSLLM):
Returns:
dict: The request body to use for the Bedrock Image Generation API
"""
provider = model.split(".")[0]
# Use the existing ARN-aware provider detection method
bedrock_provider = self.get_bedrock_invoke_provider(model)
if bedrock_provider == "amazon" or bedrock_provider == "nova":
# Handle Amazon Nova Canvas models
provider = "amazon"
elif bedrock_provider == "stability":
provider = "stability"
else:
# Fallback to original logic for backward compatibility
provider = model.split(".")[0]
inference_params = copy.deepcopy(optional_params)
inference_params.pop(
"user", None

33
litellm/llms/custom_httpx/http_handler.py
View file

@ -167,6 +167,7 @@ class AsyncHTTPHandler:
concurrent_limit=1000,
client_alias: Optional[str] = None, # name for client in logs
ssl_verify: Optional[VerifyTypes] = None,
shared_session: Optional["ClientSession"] = None,
):
self.timeout = timeout
self.event_hooks = event_hooks
@ -175,6 +176,7 @@ class AsyncHTTPHandler:
concurrent_limit=concurrent_limit,
event_hooks=event_hooks,
ssl_verify=ssl_verify,
shared_session=shared_session,
)
self.client_alias = client_alias
@ -184,6 +186,7 @@ class AsyncHTTPHandler:
concurrent_limit: int,
event_hooks: Optional[Mapping[str, List[Callable[..., Any]]]],
ssl_verify: Optional[VerifyTypes] = None,
shared_session: Optional["ClientSession"] = None,
) -> httpx.AsyncClient:
# Get unified SSL configuration
ssl_config = get_ssl_configuration(ssl_verify)
@ -199,6 +202,7 @@ class AsyncHTTPHandler:
transport = AsyncHTTPHandler._create_async_transport(
ssl_context=ssl_config if isinstance(ssl_config, ssl.SSLContext) else None,
ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
shared_session=shared_session,
)
return httpx.AsyncClient(
@ -260,7 +264,6 @@ class AsyncHTTPHandler:
files: Optional[RequestFiles] = None,
content: Any = None,
):
start_time = time.time()
try:
if timeout is None:
@ -523,7 +526,9 @@ class AsyncHTTPHandler:
@staticmethod
def _create_async_transport(
ssl_context: Optional[ssl.SSLContext] = None, ssl_verify: Optional[bool] = None
ssl_context: Optional[ssl.SSLContext] = None,
ssl_verify: Optional[bool] = None,
shared_session: Optional["ClientSession"] = None,
) -> Optional[Union[LiteLLMAiohttpTransport, AsyncHTTPTransport]]:
"""
- Creates a transport for httpx.AsyncClient
@ -544,7 +549,9 @@ class AsyncHTTPHandler:
#########################################################
if AsyncHTTPHandler._should_use_aiohttp_transport():
return AsyncHTTPHandler._create_aiohttp_transport(
ssl_context=ssl_context, ssl_verify=ssl_verify
ssl_context=ssl_context,
ssl_verify=ssl_verify,
shared_session=shared_session,
)
#########################################################
@ -612,6 +619,7 @@ class AsyncHTTPHandler:
def _create_aiohttp_transport(
ssl_verify: Optional[bool] = None,
ssl_context: Optional[ssl.SSLContext] = None,
shared_session: Optional["ClientSession"] = None,
) -> LiteLLMAiohttpTransport:
"""
Creates an AiohttpTransport with RequestNotRead error handling
@ -635,6 +643,18 @@ class AsyncHTTPHandler:
trust_env = True
verbose_logger.debug("Creating AiohttpTransport...")
# Use shared session if provided and valid
if shared_session is not None and not shared_session.closed:
verbose_logger.debug(
f"SHARED SESSION: Reusing existing ClientSession (ID: {id(shared_session)})"
)
return LiteLLMAiohttpTransport(client=shared_session)
# Create new session only if none provided or existing one is invalid
verbose_logger.debug(
"NEW SESSION: Creating new ClientSession (no shared session provided)"
)
return LiteLLMAiohttpTransport(
client=lambda: ClientSession(
connector=TCPConnector(**connector_kwargs),
@ -915,12 +935,13 @@ class HTTPHandler:
if litellm.force_ipv4:
return HTTPTransport(local_address="0.0.0.0")
else:
return None
return getattr(litellm, 'sync_transport', None)
def get_async_httpx_client(
llm_provider: Union[LlmProviders, httpxSpecialProvider],
params: Optional[dict] = None,
shared_session: Optional["ClientSession"] = None,
) -> AsyncHTTPHandler:
"""
Retrieves the async HTTP client from the cache
@ -942,10 +963,12 @@ def get_async_httpx_client(
return _cached_client
if params is not None:
params["shared_session"] = shared_session
_new_client = AsyncHTTPHandler(**params)
else:
_new_client = AsyncHTTPHandler(
timeout=httpx.Timeout(timeout=600.0, connect=5.0)
timeout=httpx.Timeout(timeout=600.0, connect=5.0),
shared_session=shared_session,
)
litellm.in_memory_llm_clients_cache.set_cache(

23
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -88,6 +88,7 @@ from litellm.utils import (
)
if TYPE_CHECKING:
from aiohttp import ClientSession
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
@ -236,11 +237,16 @@ class BaseLLMHTTPHandler:
client: Optional[AsyncHTTPHandler] = None,
json_mode: bool = False,
signed_json_body: Optional[bytes] = None,
shared_session: Optional["ClientSession"] = None,
):
if client is None:
verbose_logger.debug(
f"Creating HTTP client with shared_session: {id(shared_session) if shared_session else None}"
)
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
shared_session=shared_session,
)
else:
async_httpx_client = client
@ -290,6 +296,7 @@ class BaseLLMHTTPHandler:
headers: Optional[Dict[str, Any]] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
provider_config: Optional[BaseConfig] = None,
shared_session: Optional["ClientSession"] = None,
):
json_mode: bool = optional_params.pop("json_mode", False)
extra_body: Optional[dict] = optional_params.pop("extra_body", None)
@ -469,7 +476,7 @@ class BaseLLMHTTPHandler:
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client(
params={"ssl_verify": litellm_params.get("ssl_verify", None)}
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
)
else:
sync_httpx_client = client
@ -2283,7 +2290,7 @@ class BaseLLMHTTPHandler:
e=e,
provider_config=provider_config,
)
# Store the upload URL in litellm_params for the transformation method
litellm_params_with_url = dict(litellm_params)
litellm_params_with_url["upload_url"] = api_base
@ -2574,11 +2581,11 @@ class BaseLLMHTTPHandler:
"url": transformed_request["url"],
"headers": transformed_request["headers"],
}
# Only add data for non-GET requests
if method != "get" and transformed_request.get("data") is not None:
request_kwargs["data"] = transformed_request["data"]
batch_response = getattr(sync_httpx_client, method)(**request_kwargs)
elif isinstance(transformed_request, dict) and api_base:
# For other providers that use JSON requests
@ -2743,12 +2750,14 @@ class BaseLLMHTTPHandler:
"url": transformed_request["url"],
"headers": transformed_request["headers"],
}
# Only add data for non-GET requests
if method != "get" and transformed_request.get("data") is not None:
request_kwargs["data"] = transformed_request["data"]
batch_response = await getattr(async_httpx_client, method)(**request_kwargs)
batch_response = await getattr(async_httpx_client, method)(
**request_kwargs
)
elif isinstance(transformed_request, dict) and api_base:
# For other providers that use JSON requests
batch_response = await async_httpx_client.get(

3
litellm/llms/gemini/google_genai/transformation.py
View file

@ -121,7 +121,8 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
default_headers = {
"Content-Type": "application/json",
}
gemini_api_key = self._get_google_ai_studio_api_key(dict(litellm_params or {}))
# Use the passed api_key first, then fall back to litellm_params and environment
gemini_api_key = api_key or self._get_google_ai_studio_api_key(dict(litellm_params or {}))
if gemini_api_key is not None:
default_headers[self.XGOOGLE_API_KEY] = gemini_api_key
if headers is not None:

115
litellm/llms/gemini/image_generation/transformation.py
View file

@ -85,17 +85,25 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
) -> str:
"""
Get the complete url for the request
Google AI API format: https://generativelanguage.googleapis.com/v1beta/models/{model}:predict
Gemini 2.5 Flash Image Preview: :generateContent
Other Imagen models: :predict
"""
complete_url: str = (
api_base
or get_secret_str("GEMINI_API_BASE")
api_base
or get_secret_str("GEMINI_API_BASE")
or self.DEFAULT_BASE_URL
)
complete_url = complete_url.rstrip("/")
complete_url = f"{complete_url}/models/{model}:predict"
# Gemini 2.5 Flash Image Preview uses generateContent endpoint
if "2.5-flash-image-preview" in model:
complete_url = f"{complete_url}/models/{model}:generateContent"
else:
# All other Imagen models use predict endpoint
complete_url = f"{complete_url}/models/{model}:predict"
return complete_url
def validate_environment(
@ -128,35 +136,52 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
headers: dict,
) -> dict:
"""
Transform the image generation request to Google AI Imagen format
Google AI API format:
Transform the image generation request to Gemini format
For Gemini 2.5 Flash Image Preview, use the standard Gemini format with response_modalities:
{
"instances": [
"contents": [
{
"prompt": "Robot holding a red skateboard"
"parts": [
{"text": "Generate an image of..."}
]
}
],
"parameters": {
"sampleCount": 4,
"aspectRatio": "1:1",
"personGeneration": "allow_adult"
"generationConfig": {
"response_modalities": ["IMAGE", "TEXT"]
}
}
"""
from litellm.types.llms.gemini import (
GeminiImageGenerationInstance,
GeminiImageGenerationParameters,
)
request_body: GeminiImageGenerationRequest = GeminiImageGenerationRequest(
instances=[
GeminiImageGenerationInstance(
prompt=prompt
)
],
parameters=GeminiImageGenerationParameters(**optional_params)
)
return request_body.model_dump(exclude_none=True)
# For Gemini 2.5 Flash Image Preview, use standard Gemini format
if "2.5-flash-image-preview" in model:
request_body: dict = {
"contents": [
{
"parts": [
{"text": prompt}
]
}
],
"generationConfig": {
"response_modalities": ["IMAGE", "TEXT"]
}
}
return request_body
else:
# For other Imagen models, use the original Imagen format
from litellm.types.llms.gemini import (
GeminiImageGenerationInstance,
GeminiImageGenerationParameters,
)
request_body_obj: GeminiImageGenerationRequest = GeminiImageGenerationRequest(
instances=[
GeminiImageGenerationInstance(
prompt=prompt
)
],
parameters=GeminiImageGenerationParameters(**optional_params)
)
return request_body_obj.model_dump(exclude_none=True)
def transform_image_generation_response(
self,
@ -185,14 +210,30 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
if not model_response.data:
model_response.data = []
# Google AI returns predictions with generated images
predictions = response_data.get("predictions", [])
for prediction in predictions:
# Google AI returns base64 encoded images in the prediction
model_response.data.append(ImageObject(
b64_json=prediction.get("bytesBase64Encoded", None),
url=None, # Google AI returns base64, not URLs
))
# Handle different response formats based on model
if "2.5-flash-image-preview" in model:
# Gemini 2.5 Flash Image Preview returns in candidates format
candidates = response_data.get("candidates", [])
for candidate in candidates:
content = candidate.get("content", {})
parts = content.get("parts", [])
for part in parts:
# Look for inlineData with image
if "inlineData" in part:
inline_data = part["inlineData"]
if "data" in inline_data:
model_response.data.append(ImageObject(
b64_json=inline_data["data"],
url=None,
))
else:
# Original Imagen format - predictions with generated images
predictions = response_data.get("predictions", [])
for prediction in predictions:
# Google AI returns base64 encoded images in the prediction
model_response.data.append(ImageObject(
b64_json=prediction.get("bytesBase64Encoded", None),
url=None, # Google AI returns base64, not URLs
))
return model_response

18
litellm/llms/openai/common_utils.py
View file

@ -5,12 +5,15 @@ Common helpers / utils across al OpenAI endpoints
import hashlib
import json
import ssl
from typing import Any, Dict, List, Literal, Optional, Union
from typing import Any, Dict, List, Literal, Optional, TYPE_CHECKING, Union
import httpx
import openai
from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
if TYPE_CHECKING:
from aiohttp import ClientSession
import litellm
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
@ -194,7 +197,9 @@ class BaseOpenAILLM:
return param_names
@staticmethod
def _get_async_http_client() -> Optional[httpx.AsyncClient]:
def _get_async_http_client(
shared_session: Optional["ClientSession"] = None,
) -> Optional[httpx.AsyncClient]:
if litellm.aclient_session is not None:
return litellm.aclient_session
@ -205,8 +210,11 @@ class BaseOpenAILLM:
limits=httpx.Limits(max_connections=1000, max_keepalive_connections=100),
verify=ssl_config,
transport=AsyncHTTPHandler._create_async_transport(
ssl_context=ssl_config if isinstance(ssl_config, ssl.SSLContext) else None,
ssl_context=ssl_config
if isinstance(ssl_config, ssl.SSLContext)
else None,
ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
shared_session=shared_session,
),
follow_redirects=True,
)
@ -215,10 +223,10 @@ class BaseOpenAILLM:
def _get_sync_http_client() -> Optional[httpx.Client]:
if litellm.client_session is not None:
return litellm.client_session
# Get unified SSL configuration
ssl_config = get_ssl_configuration()
return httpx.Client(
limits=httpx.Limits(max_connections=1000, max_keepalive_connections=100),
verify=ssl_config,

4
litellm/llms/openai/cost_calculation.py
View file

@ -18,7 +18,7 @@ def cost_router(call_type: CallTypes) -> Literal["cost_per_token", "cost_per_sec
return "cost_per_token"
def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
def cost_per_token(model: str, usage: Usage, service_tier: Optional[str] = None) -> Tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@ -31,7 +31,7 @@ def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
"""
## CALCULATE INPUT COST
return generic_cost_per_token(
model=model, usage=usage, custom_llm_provider="openai"
model=model, usage=usage, custom_llm_provider="openai", service_tier=service_tier
)
# ### Non-cached text tokens
# non_cached_text_tokens = usage.prompt_tokens

15
litellm/llms/openai/openai.py
View file

@ -10,12 +10,16 @@ from typing import (
List,
Literal,
Optional,
TYPE_CHECKING,
Union,
cast,
)
from urllib.parse import urlparse
import httpx
if TYPE_CHECKING:
from aiohttp import ClientSession
import openai
from openai import AsyncOpenAI, OpenAI
from openai.types.beta.assistant_deleted import AssistantDeleted
@ -355,6 +359,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
max_retries: Optional[int] = DEFAULT_MAX_RETRIES,
organization: Optional[str] = None,
client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
shared_session: Optional["ClientSession"] = None,
) -> Optional[Union[OpenAI, AsyncOpenAI]]:
client_initialization_params: Dict = locals()
if client is None:
@ -379,7 +384,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
_new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI(
api_key=api_key,
base_url=api_base,
http_client=OpenAIChatCompletion._get_async_http_client(),
http_client=OpenAIChatCompletion._get_async_http_client(
shared_session=shared_session
),
timeout=timeout,
max_retries=max_retries,
organization=organization,
@ -522,8 +529,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
organization: Optional[str] = None,
custom_llm_provider: Optional[str] = None,
drop_params: Optional[bool] = None,
shared_session: Optional["ClientSession"] = None,
):
super().completion()
super().completion(shared_session=shared_session)
try:
fake_stream: bool = False
inference_params = optional_params.copy()
@ -606,6 +614,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
organization=organization,
drop_params=drop_params,
fake_stream=fake_stream,
shared_session=shared_session,
)
data = provider_config.transform_request(
@ -771,6 +780,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
drop_params: Optional[bool] = None,
stream_options: Optional[dict] = None,
fake_stream: bool = False,
shared_session: Optional["ClientSession"] = None,
):
response = None
data = await provider_config.async_transform_request(
@ -793,6 +803,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
max_retries=max_retries,
organization=organization,
client=client,
shared_session=shared_session,
)
## LOGGING

9
litellm/llms/vertex_ai/batches/transformation.py
View file

@ -114,7 +114,14 @@ class VertexAIBatchTransformation:
"""
Gets the output file id from the Vertex AI Batch response
"""
output_file_id: str = ""
output_file_id: str = (
response.get("outputInfo", OutputInfo()).get("gcsOutputDirectory", "")
+ "/predictions.jsonl"
)
if output_file_id != "/predictions.jsonl":
return output_file_id
output_config = response.get("outputConfig")
if output_config is None:
return output_file_id

143
litellm/llms/vertex_ai/files/handler.py
View file

@ -1,5 +1,6 @@
import asyncio
from typing import Any, Coroutine, Optional, Union
import urllib.parse
from typing import Any, Coroutine, Optional, Tuple, Union
import httpx
@ -9,7 +10,12 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import (
GCSLoggingConfig,
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.openai import CreateFileRequest, OpenAIFileObject
from litellm.types.llms.openai import (
CreateFileRequest,
FileContentRequest,
HttpxBinaryResponseContent,
OpenAIFileObject,
)
from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
from .transformation import VertexAIJsonlFilesTransformation
@ -105,3 +111,136 @@ class VertexAIFilesHandler(GCSBucketBase):
max_retries=max_retries,
)
)
def _extract_bucket_and_object_from_file_id(self, file_id: str) -> Tuple[str, str]:
"""
Extract bucket name and object path from URL-encoded file_id.
Expected format: gs%3A%2F%2Fbucket-name%2Fpath%2Fto%2Ffile
Which decodes to: gs://bucket-name/path/to/file
Returns:
tuple: (bucket_name, url_encoded_object_path)
- bucket_name: "bucket-name"
- url_encoded_object_path: "path%2Fto%2Ffile"
"""
decoded_path = urllib.parse.unquote(file_id)
if decoded_path.startswith("gs://"):
full_path = decoded_path[5:] # Remove 'gs://' prefix
else:
full_path = decoded_path
if "/" in full_path:
bucket_name, object_path = full_path.split("/", 1)
else:
bucket_name = full_path
object_path = ""
encoded_object_path = urllib.parse.quote(object_path, safe="")
return bucket_name, encoded_object_path
async def afile_content(
self,
file_content_request: FileContentRequest,
vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES],
vertex_project: Optional[str],
vertex_location: Optional[str],
timeout: Union[float, httpx.Timeout],
max_retries: Optional[int],
) -> HttpxBinaryResponseContent:
"""
Download file content from GCS bucket for VertexAI files.
Args:
file_content_request: Contains file_id (URL-encoded GCS path)
vertex_credentials: VertexAI credentials
vertex_project: VertexAI project ID
vertex_location: VertexAI location
timeout: Request timeout
max_retries: Max retry attempts
Returns:
HttpxBinaryResponseContent: Binary content wrapped in compatible response format
"""
file_id = file_content_request.get("file_id")
if not file_id:
raise ValueError("file_id is required in file_content_request")
bucket_name, encoded_object_path = self._extract_bucket_and_object_from_file_id(
file_id
)
download_kwargs = {
"standard_callback_dynamic_params": {"gcs_bucket_name": bucket_name}
}
file_content = await self.download_gcs_object(
object_name=encoded_object_path, **download_kwargs
)
if file_content is None:
decoded_path = urllib.parse.unquote(file_id)
raise ValueError(f"Failed to download file from GCS: {decoded_path}")
decoded_path = urllib.parse.unquote(file_id)
mock_response = httpx.Response(
status_code=200,
content=file_content,
headers={"content-type": "application/octet-stream"},
request=httpx.Request(method="GET", url=decoded_path),
)
return HttpxBinaryResponseContent(response=mock_response)
def file_content(
self,
_is_async: bool,
file_content_request: FileContentRequest,
api_base: Optional[str],
vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES],
vertex_project: Optional[str],
vertex_location: Optional[str],
timeout: Union[float, httpx.Timeout],
max_retries: Optional[int],
) -> Union[
HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent]
]:
"""
Download file content from GCS bucket for VertexAI files.
Supports both sync and async operations.
Args:
_is_async: Whether to run asynchronously
file_content_request: Contains file_id (URL-encoded GCS path)
api_base: API base (unused for GCS operations)
vertex_credentials: VertexAI credentials
vertex_project: VertexAI project ID
vertex_location: VertexAI location
timeout: Request timeout
max_retries: Max retry attempts
Returns:
HttpxBinaryResponseContent or Coroutine: Binary content wrapped in compatible response format
"""
if _is_async:
return self.afile_content(
file_content_request=file_content_request,
vertex_credentials=vertex_credentials,
vertex_project=vertex_project,
vertex_location=vertex_location,
timeout=timeout,
max_retries=max_retries,
)
else:
return asyncio.run(
self.afile_content(
file_content_request=file_content_request,
vertex_credentials=vertex_credentials,
vertex_project=vertex_project,
vertex_location=vertex_location,
timeout=timeout,
max_retries=max_retries,
)
)

6
litellm/llms/vertex_ai/files/transformation.py
View file

@ -261,10 +261,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
raise ValueError("file is required")
extracted_file_data = extract_file_data(file_data)
extracted_file_data_content = extracted_file_data.get("content")
if extracted_file_data_content is None:
raise ValueError("file content is required")
if FilesAPIUtils.is_batch_jsonl_file(
create_file_data=create_file_data,
extracted_file_data=extracted_file_data,
@ -283,7 +283,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
openai_jsonl_content
)
)
return json.dumps(vertex_jsonl_content)
return "\n".join(json.dumps(item) for item in vertex_jsonl_content)
elif isinstance(extracted_file_data_content, bytes):
return extracted_file_data_content
else:

14
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -537,7 +537,11 @@ def sync_transform_request_body(
logging_obj=logging_obj,
)
else: # [TODO] implement context caching for gemini as well
cached_content = optional_params.pop("cached_content", None)
cached_content = None
if "cached_content" in optional_params:
cached_content = optional_params.pop("cached_content")
elif "cachedContent" in optional_params:
cached_content = optional_params.pop("cachedContent")
return _transform_request_body(
messages=messages,
@ -584,7 +588,11 @@ async def async_transform_request_body(
logging_obj=logging_obj,
)
else: # [TODO] implement context caching for gemini as well
cached_content = optional_params.pop("cached_content", None)
cached_content = None
if "cached_content" in optional_params:
cached_content = optional_params.pop("cached_content")
elif "cachedContent" in optional_params:
cached_content = optional_params.pop("cachedContent")
return _transform_request_body(
messages=messages,
@ -649,5 +657,3 @@ def _transform_system_message(
return SystemInstructions(parts=system_content_blocks), messages
return None, messages

10
litellm/llms/vertex_ai/vertex_llm_base.py
View file

@ -271,17 +271,11 @@ class VertexBase:
def is_using_v1beta1_features(self, optional_params: dict) -> bool:
"""
VertexAI only supports ContextCaching on v1beta1
use this helper to decide if request should be sent to v1 or v1beta1
Returns v1beta1 if context caching is enabled
Returns v1 in all other cases
Returns true if any beta feature is enabled
Returns false in all other cases
"""
if "cached_content" in optional_params:
return True
if "CachedContent" in optional_params:
return True
return False
def _check_custom_proxy(

21
litellm/llms/vllm/common_utils.py
View file

@ -11,7 +11,21 @@ from litellm.utils import _add_path_to_api_base
class VLLMError(BaseLLMException):
pass
def __init__(
self,
status_code: int,
message: str,
request: Optional[httpx.Request] = None,
response: Optional[httpx.Response] = None,
headers: Optional[Union[httpx.Headers, dict]] = None,
):
super().__init__(
status_code=status_code,
message=message,
request=request,
response=response,
headers=headers,
)
class VLLMModelInfo(BaseLLMModelInfo):
@ -25,7 +39,8 @@ class VLLMModelInfo(BaseLLMModelInfo):
api_key: Optional[str] = None,
api_base: Optional[str] = None,
) -> dict:
"""Google AI Studio sends api key in query params"""
if api_key is not None:
headers["x-api-key"] = api_key
return headers
@staticmethod
@ -53,7 +68,7 @@ class VLLMModelInfo(BaseLLMModelInfo):
endpoint = "/v1/models"
if api_base is None or api_key is None:
raise ValueError(
"GEMINI_API_BASE or GEMINI_API_KEY is not set. Please set the environment variable, to query Gemini's `/models` endpoint."
"VLLM_API_BASE or VLLM_API_KEY is not set. Please set the environment variable, to query VLLM's `/models` endpoint."
)
url = _add_path_to_api_base(api_base, endpoint)

0
litellm/llms/wandb/__init__.py Normal file
View file

0
litellm/llms/wandb/chat/__init__.py Normal file
View file

27
litellm/llms/wandb/chat/transformation.py Normal file
View file

@ -0,0 +1,27 @@
"""
Wandb Chat Completions API - Transformation
This is OpenAI compatible - no translation needed / occurs
"""
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
class WandbConfig(OpenAIGPTConfig):
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
"""
map max_completion_tokens param to max_tokens
"""
supported_openai_params = self.get_supported_openai_params(model=model)
for param, value in non_default_params.items():
if param == "max_completion_tokens":
optional_params["max_tokens"] = value
elif param in supported_openai_params:
optional_params[param] = value
return optional_params

65
litellm/main.py
View file

@ -36,8 +36,12 @@ from typing import (
Union,
cast,
get_args,
TYPE_CHECKING,
)
if TYPE_CHECKING:
from aiohttp import ClientSession
import dotenv
import httpx
import openai
@ -374,6 +378,8 @@ async def acompletion(
# Optional liteLLM function params
thinking: Optional[AnthropicThinkingParam] = None,
web_search_options: Optional[OpenAIWebSearchOptions] = None,
# Session management
shared_session: Optional["ClientSession"] = None,
**kwargs,
) -> Union[ModelResponse, CustomStreamWrapper]:
"""
@ -466,6 +472,16 @@ async def acompletion(
#########################################################
#########################################################
# Log shared session usage
if shared_session is not None:
verbose_logger.debug(
f"🔄 SHARED SESSION: acompletion called with shared_session (ID: {id(shared_session)})"
)
else:
verbose_logger.debug(
"🔄 NO SHARED SESSION: acompletion called without shared_session"
)
# Adjusted to use explicit arguments instead of *args and **kwargs
completion_kwargs = {
"model": model,
@ -506,6 +522,7 @@ async def acompletion(
"acompletion": True, # assuming this is a required parameter
"thinking": thinking,
"web_search_options": web_search_options,
"shared_session": shared_session,
}
if custom_llm_provider is None:
_, custom_llm_provider, _, _ = get_llm_provider(
@ -930,6 +947,8 @@ def completion( # type: ignore # noqa: PLR0915
model_list: Optional[list] = None, # pass in a list of api_base,keys, etc.
# Optional liteLLM function params
thinking: Optional[AnthropicThinkingParam] = None,
# Session management
shared_session: Optional["ClientSession"] = None,
**kwargs,
) -> Union[ModelResponse, CustomStreamWrapper]:
"""
@ -1596,6 +1615,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout, # type: ignore
client=client,
custom_llm_provider=custom_llm_provider,
@ -1642,6 +1662,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout, # type: ignore
client=client, # pass AsyncOpenAI, OpenAI client
custom_llm_provider=custom_llm_provider,
@ -1771,6 +1792,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout, # type: ignore
client=client,
custom_llm_provider=custom_llm_provider,
@ -1800,6 +1822,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
@ -1830,6 +1853,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout, # type: ignore
client=client,
custom_llm_provider=custom_llm_provider,
@ -1881,6 +1905,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider=custom_llm_provider,
timeout=timeout,
headers=headers,
@ -1954,6 +1979,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
@ -1980,6 +2006,7 @@ def completion( # type: ignore # noqa: PLR0915
or custom_llm_provider == "openai"
or custom_llm_provider == "together_ai"
or custom_llm_provider == "nebius"
or custom_llm_provider == "wandb"
or custom_llm_provider in litellm.openai_compatible_providers
or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo
): # allow user to make an openai call with a custom base
@ -2044,6 +2071,7 @@ def completion( # type: ignore # noqa: PLR0915
optional_params=optional_params,
timeout=timeout,
litellm_params=litellm_params,
shared_session=shared_session,
acompletion=acompletion,
stream=stream,
api_key=api_key,
@ -2070,6 +2098,7 @@ def completion( # type: ignore # noqa: PLR0915
client=client, # pass AsyncOpenAI, OpenAI client
organization=organization,
custom_llm_provider=custom_llm_provider,
shared_session=shared_session,
)
except Exception as e:
## LOGGING - log the original exception returned
@ -2110,6 +2139,7 @@ def completion( # type: ignore # noqa: PLR0915
optional_params=optional_params,
timeout=timeout,
litellm_params=litellm_params,
shared_session=shared_session,
acompletion=acompletion,
stream=stream,
api_key=api_key,
@ -2197,6 +2227,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="clarifai",
timeout=timeout,
headers=headers,
@ -2242,6 +2273,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="anthropic_text",
timeout=timeout,
headers=headers,
@ -2427,6 +2459,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="cohere_chat",
timeout=timeout,
headers=headers,
@ -2694,6 +2727,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="openrouter",
timeout=timeout,
headers=headers,
@ -2756,6 +2790,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="vercel_ai_gateway",
timeout=timeout,
headers=headers,
@ -3242,6 +3277,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="watsonx_text",
timeout=timeout,
headers=headers,
@ -3295,6 +3331,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="ollama",
timeout=timeout,
headers=headers,
@ -3328,6 +3365,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="ollama_chat",
timeout=timeout,
headers=headers,
@ -3348,6 +3386,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider=custom_llm_provider,
timeout=timeout,
headers=headers,
@ -3380,6 +3419,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="cloudflare",
timeout=timeout,
headers=headers,
@ -3433,6 +3473,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
timeout=timeout, # type: ignore
client=client,
custom_llm_provider=custom_llm_provider,
@ -3461,6 +3502,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
shared_session=shared_session,
custom_llm_provider="gradient_ai",
timeout=timeout,
headers=headers,
@ -3614,7 +3656,7 @@ def completion( # type: ignore # noqa: PLR0915
async_fn=acompletion, stream=stream, custom_llm=custom_handler
)
headers = headers or litellm.headers
headers = headers or litellm.headers or {}
## CALL FUNCTION
response = handler_fn(
@ -4412,6 +4454,27 @@ def embedding( # noqa: PLR0915
or "api.studio.nebius.ai/v1"
)
response = openai_chat_completions.embedding(
model=model,
input=input,
api_base=api_base,
api_key=api_key,
logging_obj=logging,
timeout=timeout,
model_response=EmbeddingResponse(),
optional_params=optional_params,
client=client,
aembedding=aembedding,
)
elif custom_llm_provider == "wandb":
api_key = api_key or litellm.api_key or get_secret_str("WANDB_API_KEY")
api_base = (
api_base
or litellm.api_base
or get_secret_str("WANDB_API_BASE")
or "https://api.inference.wandb.ai/v1"
)
response = openai_chat_completions.embedding(
model=model,
input=input,

338
litellm/model_prices_and_context_window_backup.json
View file

@ -560,10 +560,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@ -824,10 +828,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@ -4787,10 +4795,14 @@
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "anthropic",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@ -7547,10 +7559,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@ -9126,7 +9142,7 @@
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"mode": "image_generation",
"output_cost_per_image": 0.039,
"output_cost_per_reasoning_token": 3e-05,
"output_cost_per_token": 3e-05,
@ -10489,7 +10505,7 @@
"max_tokens": 65535,
"max_video_length": 1,
"max_videos_per_prompt": 10,
"mode": "chat",
"mode": "image_generation",
"output_cost_per_image": 0.039,
"output_cost_per_reasoning_token": 3e-05,
"output_cost_per_token": 3e-05,
@ -11534,8 +11550,10 @@
},
"gpt-4.1": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_priority": 8.75e-07,
"input_cost_per_token": 2e-06,
"input_cost_per_token_batches": 1e-06,
"input_cost_per_token_priority": 3.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
@ -11543,6 +11561,7 @@
"mode": "chat",
"output_cost_per_token": 8e-06,
"output_cost_per_token_batches": 4e-06,
"output_cost_per_token_priority": 1.4e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -11600,8 +11619,10 @@
},
"gpt-4.1-mini": {
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_priority": 1.75e-07,
"input_cost_per_token": 4e-07,
"input_cost_per_token_batches": 2e-07,
"input_cost_per_token_priority": 7e-07,
"litellm_provider": "openai",
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
@ -11609,6 +11630,7 @@
"mode": "chat",
"output_cost_per_token": 1.6e-06,
"output_cost_per_token_batches": 8e-07,
"output_cost_per_token_priority": 2.8e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -11666,8 +11688,10 @@
},
"gpt-4.1-nano": {
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_priority": 5e-08,
"input_cost_per_token": 1e-07,
"input_cost_per_token_batches": 5e-08,
"input_cost_per_token_priority": 2e-07,
"litellm_provider": "openai",
"max_input_tokens": 1047576,
"max_output_tokens": 32768,
@ -11675,6 +11699,7 @@
"mode": "chat",
"output_cost_per_token": 4e-07,
"output_cost_per_token_batches": 2e-07,
"output_cost_per_token_priority": 8e-07,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -11773,8 +11798,10 @@
},
"gpt-4o": {
"cache_read_input_token_cost": 1.25e-06,
"cache_read_input_token_cost_priority": 2.125e-06,
"input_cost_per_token": 2.5e-06,
"input_cost_per_token_batches": 1.25e-06,
"input_cost_per_token_priority": 4.25e-06,
"litellm_provider": "openai",
"max_input_tokens": 128000,
"max_output_tokens": 16384,
@ -11782,6 +11809,7 @@
"mode": "chat",
"output_cost_per_token": 1e-05,
"output_cost_per_token_batches": 5e-06,
"output_cost_per_token_priority": 1.7e-05,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
@ -11794,6 +11822,7 @@
"gpt-4o-2024-05-13": {
"input_cost_per_token": 5e-06,
"input_cost_per_token_batches": 2.5e-06,
"input_cost_per_token_priority": 8.75e-06,
"litellm_provider": "openai",
"max_input_tokens": 128000,
"max_output_tokens": 4096,
@ -11801,6 +11830,7 @@
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"output_cost_per_token_priority": 2.625e-05,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
@ -11919,8 +11949,10 @@
},
"gpt-4o-mini": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_priority": 1.25e-07,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7.5e-08,
"input_cost_per_token_priority": 2.5e-07,
"litellm_provider": "openai",
"max_input_tokens": 128000,
"max_output_tokens": 16384,
@ -11928,6 +11960,7 @@
"mode": "chat",
"output_cost_per_token": 6e-07,
"output_cost_per_token_batches": 3e-07,
"output_cost_per_token_priority": 1e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
@ -12243,13 +12276,19 @@
},
"gpt-5": {
"cache_read_input_token_cost": 1.25e-07,
"cache_read_input_token_cost_flex": 6.25e-08,
"cache_read_input_token_cost_priority": 2.5e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_flex": 6.25e-07,
"input_cost_per_token_priority": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1e-05,
"output_cost_per_token_flex": 5e-06,
"output_cost_per_token_priority": 2e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -12275,13 +12314,19 @@
},
"gpt-5-2025-08-07": {
"cache_read_input_token_cost": 1.25e-07,
"cache_read_input_token_cost_flex": 6.25e-08,
"cache_read_input_token_cost_priority": 2.5e-07,
"input_cost_per_token": 1.25e-06,
"input_cost_per_token_flex": 6.25e-07,
"input_cost_per_token_priority": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1e-05,
"output_cost_per_token_flex": 5e-06,
"output_cost_per_token_priority": 2e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -12303,6 +12348,7 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_service_tier": true,
"supports_vision": true
},
"gpt-5-chat": {
@ -12371,13 +12417,19 @@
},
"gpt-5-mini": {
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
"cache_read_input_token_cost_priority": 4.5e-08,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_flex": 1.25e-07,
"input_cost_per_token_priority": 4.5e-07,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2e-06,
"output_cost_per_token_flex": 1e-06,
"output_cost_per_token_priority": 3.6e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -12403,13 +12455,19 @@
},
"gpt-5-mini-2025-08-07": {
"cache_read_input_token_cost": 2.5e-08,
"cache_read_input_token_cost_flex": 1.25e-08,
"cache_read_input_token_cost_priority": 4.5e-08,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_flex": 1.25e-07,
"input_cost_per_token_priority": 4.5e-07,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2e-06,
"output_cost_per_token_flex": 1e-06,
"output_cost_per_token_priority": 3.6e-06,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -12435,13 +12493,17 @@
},
"gpt-5-nano": {
"cache_read_input_token_cost": 5e-09,
"cache_read_input_token_cost_flex": 2.5e-09,
"input_cost_per_token": 5e-08,
"input_cost_per_token_flex": 2.5e-08,
"input_cost_per_token_priority": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4e-07,
"output_cost_per_token_flex": 2e-07,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -12467,13 +12529,16 @@
},
"gpt-5-nano-2025-08-07": {
"cache_read_input_token_cost": 5e-09,
"cache_read_input_token_cost_flex": 2.5e-09,
"input_cost_per_token": 5e-08,
"input_cost_per_token_flex": 2.5e-08,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 4e-07,
"output_cost_per_token_flex": 2e-07,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/batch",
@ -15177,13 +15242,19 @@
},
"o3": {
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_flex": 2.5e-07,
"cache_read_input_token_cost_priority": 8.75e-07,
"input_cost_per_token": 2e-06,
"input_cost_per_token_flex": 1e-06,
"input_cost_per_token_priority": 3.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 200000,
"max_output_tokens": 100000,
"max_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 8e-06,
"output_cost_per_token_flex": 4e-06,
"output_cost_per_token_priority": 1.4e-05,
"supported_endpoints": [
"/v1/responses",
"/v1/chat/completions",
@ -15399,13 +15470,19 @@
},
"o4-mini": {
"cache_read_input_token_cost": 2.75e-07,
"cache_read_input_token_cost_flex": 1.375e-07,
"cache_read_input_token_cost_priority": 5e-07,
"input_cost_per_token": 1.1e-06,
"input_cost_per_token_flex": 5.5e-07,
"input_cost_per_token_priority": 2e-06,
"litellm_provider": "openai",
"max_input_tokens": 200000,
"max_output_tokens": 100000,
"max_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 4.4e-06,
"output_cost_per_token_flex": 2.2e-06,
"output_cost_per_token_priority": 8e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": false,
"supports_pdf_input": true,
@ -16096,10 +16173,12 @@
"openrouter/anthropic/claude-sonnet-4": {
"input_cost_per_image": 0.0048,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"litellm_provider": "openrouter",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
@ -16900,6 +16979,20 @@
"supports_tool_choice": true,
"supports_web_search": true
},
"openrouter/x-ai/grok-4-fast:free": {
"input_cost_per_token": 0,
"litellm_provider": "openrouter",
"max_input_tokens": 2000000,
"max_output_tokens": 30000,
"max_tokens": 2000000,
"mode": "chat",
"output_cost_per_token": 0,
"source": "https://openrouter.ai/x-ai/grok-4-fast:free",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_web_search": false
},
"ovhcloud/DeepSeek-R1-Distill-Llama-70B": {
"input_cost_per_token": 6.7e-07,
"litellm_provider": "ovhcloud",
@ -18945,10 +19038,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@ -20281,10 +20378,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@ -20307,10 +20408,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"input_cost_per_token_above_200k_tokens": 6e-06,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"max_tokens": 64000,
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@ -20943,6 +21048,132 @@
"mode": "embedding",
"output_cost_per_token": 0.0
},
"wandb/openai/gpt-oss-120b": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 0.015,
"output_cost_per_token": 0.06,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/openai/gpt-oss-20b": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 0.005,
"output_cost_per_token": 0.02,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/zai-org/GLM-4.5": {
"max_tokens": 131072,
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 0.055,
"output_cost_per_token": 0.2,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/Qwen/Qwen3-235B-A22B-Instruct-2507": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 0.01,
"output_cost_per_token": 0.01,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/Qwen/Qwen3-Coder-480B-A35B-Instruct": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 0.1,
"output_cost_per_token": 0.15,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/Qwen/Qwen3-235B-A22B-Thinking-2507": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"input_cost_per_token": 0.01,
"output_cost_per_token": 0.01,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/moonshotai/Kimi-K2-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.135,
"output_cost_per_token": 0.4,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/meta-llama/Llama-3.1-8B-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.022,
"output_cost_per_token": 0.022,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/deepseek-ai/DeepSeek-V3.1": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.055,
"output_cost_per_token": 0.165,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/deepseek-ai/DeepSeek-R1-0528": {
"max_tokens": 161000,
"max_input_tokens": 161000,
"max_output_tokens": 161000,
"input_cost_per_token": 0.135,
"output_cost_per_token": 0.54,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/deepseek-ai/DeepSeek-V3-0324": {
"max_tokens": 161000,
"max_input_tokens": 161000,
"max_output_tokens": 161000,
"input_cost_per_token": 0.114,
"output_cost_per_token": 0.275,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/meta-llama/Llama-3.3-70B-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.071,
"output_cost_per_token": 0.071,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
"max_tokens": 64000,
"max_input_tokens": 64000,
"max_output_tokens": 64000,
"input_cost_per_token": 0.017,
"output_cost_per_token": 0.066,
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/microsoft/Phi-4-mini-instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 128000,
"input_cost_per_token": 0.008,
"output_cost_per_token": 0.035,
"litellm_provider": "wandb",
"mode": "chat"
},
"watsonx/ibm/granite-3-8b-instruct": {
"input_cost_per_token": 0.0002,
"litellm_provider": "watsonx",
@ -20988,6 +21219,30 @@
"/v1/audio/transcriptions"
]
},
"vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "vertex_ai-qwen_models",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supports_function_calling": true,
"supports_tool_choice": true
},
"vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "vertex_ai-qwen_models",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supports_function_calling": true,
"supports_tool_choice": true
},
"xai/grok-2": {
"input_cost_per_token": 2e-06,
"litellm_provider": "xai",
@ -21240,6 +21495,35 @@
"supports_tool_choice": true,
"supports_web_search": true
},
"xai/grok-4-fast-reasoning": {
"litellm_provider": "xai",
"max_input_tokens": 2e6,
"max_output_tokens": 2e6,
"max_tokens": 2e6,
"mode": "chat",
"input_cost_per_token": 0.2e-06,
"output_cost_per_token": 0.5e-06,
"cache_read_input_token_cost": 0.05e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_web_search": true
},
"xai/grok-4-fast-non-reasoning": {
"litellm_provider": "xai",
"max_input_tokens": 2e6,
"max_output_tokens": 2e6,
"cache_read_input_token_cost": 0.05e-06,
"max_tokens": 2e6,
"mode": "chat",
"input_cost_per_token": 0.2e-06,
"output_cost_per_token": 0.5e-06,
"source": "https://docs.x.ai/docs/models",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_web_search": true
},
"xai/grok-4-0709": {
"input_cost_per_token": 3e-06,
"litellm_provider": "xai",
@ -21337,4 +21621,4 @@
"supports_vision": true,
"supports_web_search": true
}
}
}

205
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -28,12 +28,16 @@ class MCPRequestHandler:
LITELLM_MCP_SERVERS_HEADER_NAME = SpecialHeaders.mcp_servers.value
LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME = SpecialHeaders.mcp_access_groups.value
# MCP Protocol Version header
MCP_PROTOCOL_VERSION_HEADER_NAME = "MCP-Protocol-Version"
@staticmethod
async def process_mcp_request(scope: Scope) -> Tuple[UserAPIKeyAuth, Optional[str], Optional[List[str]], Optional[Dict[str, str]], Optional[str]]:
async def process_mcp_request(
scope: Scope,
) -> Tuple[
UserAPIKeyAuth, Optional[str], Optional[List[str]], Optional[Dict[str, str]]
]:
"""
Process and validate MCP request headers from the ASGI scope.
This includes:
@ -49,7 +53,6 @@ class MCPRequestHandler:
mcp_auth_header: Optional[str] MCP auth header to be passed to the MCP server (deprecated)
mcp_servers: Optional[List[str]] List of MCP servers and access groups to use
mcp_server_auth_headers: Optional[Dict[str, str]] Server-specific auth headers in format {server_alias: auth_value}
mcp_protocol_version: Optional[str] MCP protocol version from request header
Raises:
HTTPException: If headers are invalid or missing required headers
@ -58,39 +61,50 @@ class MCPRequestHandler:
litellm_api_key = (
MCPRequestHandler.get_litellm_api_key_from_headers(headers) or ""
)
# Get the old mcp_auth_header for backward compatibility
mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(headers)
# Get the new server-specific auth headers
mcp_server_auth_headers = MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
# Get MCP protocol version from header
mcp_protocol_version = headers.get(MCPRequestHandler.MCP_PROTOCOL_VERSION_HEADER_NAME)
# Get the new server-specific auth headers
mcp_server_auth_headers = (
MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
)
# Parse MCP servers from header
mcp_servers_header = headers.get(MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME)
mcp_servers_header = headers.get(
MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME
)
verbose_logger.debug(f"Raw MCP servers header: {mcp_servers_header}")
mcp_servers = None
if mcp_servers_header is not None:
try:
mcp_servers = [s.strip() for s in mcp_servers_header.split(",") if s.strip()]
mcp_servers = [
s.strip() for s in mcp_servers_header.split(",") if s.strip()
]
verbose_logger.debug(f"Parsed MCP servers: {mcp_servers}")
except Exception as e:
verbose_logger.debug(f"Error parsing mcp_servers header: {e}")
mcp_servers = None
if mcp_servers_header == "" or (mcp_servers is not None and len(mcp_servers) == 0):
if mcp_servers_header == "" or (
mcp_servers is not None and len(mcp_servers) == 0
):
mcp_servers = []
# Create a proper Request object with mock body method to avoid ASGI receive channel issues
request = Request(scope=scope)
async def mock_body():
return b"{}"
request.body = mock_body # type: ignore
validated_user_api_key_auth = await user_api_key_auth(
api_key=litellm_api_key, request=request
)
return validated_user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers, mcp_protocol_version
return (
validated_user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
)
@staticmethod
def _get_mcp_auth_header_from_headers(headers: Headers) -> Optional[str]:
@ -104,10 +118,12 @@ class MCPRequestHandler:
Support this auth: https://docs.litellm.ai/docs/mcp#using-your-mcp-with-client-side-credentials
If you want to use a different header name, you can set the `LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME` in the secret manager or `mcp_client_side_auth_header_name` in the general settings.
DEPRECATED: This method is deprecated in favor of server-specific auth headers using the format x-mcp-{{server_alias}}-{{header_name}} instead.
"""
mcp_client_side_auth_header_name: str = MCPRequestHandler._get_mcp_client_side_auth_header_name()
mcp_client_side_auth_header_name: str = (
MCPRequestHandler._get_mcp_client_side_auth_header_name()
)
auth_header = headers.get(mcp_client_side_auth_header_name)
if auth_header:
verbose_logger.warning(
@ -115,42 +131,49 @@ class MCPRequestHandler:
f"Please use server-specific auth headers in the format 'x-mcp-{{server_alias}}-{{header_name}}' instead."
)
return auth_header
@staticmethod
def _get_mcp_server_auth_headers_from_headers(headers: Headers) -> Dict[str, str]:
"""
Parse server-specific MCP auth headers from the request headers.
Looks for headers in the format: x-mcp-{server_alias}-{header_name}
Examples:
- x-mcp-github-authorization: Bearer token123
- x-mcp-zapier-x-api-key: api_key_456
- x-mcp-deepwiki-authorization: Basic base64_encoded_creds
Returns:
Dict[str, str]: Mapping of server alias to auth value
"""
server_auth_headers = {}
prefix = "x-mcp-"
for header_name, header_value in headers.items():
if header_name.lower().startswith(prefix):
# Skip the access groups header as it's not a server auth header
if header_name.lower() == MCPRequestHandler.LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME.lower() or header_name.lower() == MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME.lower():
if (
header_name.lower()
== MCPRequestHandler.LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME.lower()
or header_name.lower()
== MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME.lower()
):
continue
# Extract server_alias and header_name from x-mcp-{server_alias}-{header_name}
remaining = header_name[len(prefix):].lower()
if '-' in remaining:
remaining = header_name[len(prefix) :].lower()
if "-" in remaining:
# Split on the last dash to separate server_alias from header_name
parts = remaining.rsplit('-', 1)
parts = remaining.rsplit("-", 1)
if len(parts) == 2:
server_alias, auth_header_name = parts
server_auth_headers[server_alias] = header_value
verbose_logger.debug(f"Found server auth header: {server_alias} -> {auth_header_name}: {header_value[:10]}...")
verbose_logger.debug(
f"Found server auth header: {server_alias} -> {auth_header_name}: {header_value[:10]}..."
)
return server_auth_headers
@staticmethod
def _get_mcp_client_side_auth_header_name() -> str:
"""
@ -162,13 +185,21 @@ class MCPRequestHandler:
"""
from litellm.proxy.proxy_server import general_settings
from litellm.secret_managers.main import get_secret_str
MCP_CLIENT_SIDE_AUTH_HEADER_NAME: str = MCPRequestHandler.LITELLM_MCP_AUTH_HEADER_NAME
if get_secret_str("LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME") is not None:
MCP_CLIENT_SIDE_AUTH_HEADER_NAME = get_secret_str("LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME") or MCP_CLIENT_SIDE_AUTH_HEADER_NAME
elif general_settings.get("mcp_client_side_auth_header_name") is not None:
MCP_CLIENT_SIDE_AUTH_HEADER_NAME = general_settings.get("mcp_client_side_auth_header_name") or MCP_CLIENT_SIDE_AUTH_HEADER_NAME
return MCP_CLIENT_SIDE_AUTH_HEADER_NAME
MCP_CLIENT_SIDE_AUTH_HEADER_NAME: str = (
MCPRequestHandler.LITELLM_MCP_AUTH_HEADER_NAME
)
if get_secret_str("LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME") is not None:
MCP_CLIENT_SIDE_AUTH_HEADER_NAME = (
get_secret_str("LITELLM_MCP_CLIENT_SIDE_AUTH_HEADER_NAME")
or MCP_CLIENT_SIDE_AUTH_HEADER_NAME
)
elif general_settings.get("mcp_client_side_auth_header_name") is not None:
MCP_CLIENT_SIDE_AUTH_HEADER_NAME = (
general_settings.get("mcp_client_side_auth_header_name")
or MCP_CLIENT_SIDE_AUTH_HEADER_NAME
)
return MCP_CLIENT_SIDE_AUTH_HEADER_NAME
@staticmethod
def get_litellm_api_key_from_headers(headers: Headers) -> Optional[str]:
@ -229,10 +260,14 @@ class MCPRequestHandler:
try:
allowed_mcp_servers: List[str] = []
allowed_mcp_servers_for_key = (
await MCPRequestHandler._get_allowed_mcp_servers_for_key(user_api_key_auth)
await MCPRequestHandler._get_allowed_mcp_servers_for_key(
user_api_key_auth
)
)
allowed_mcp_servers_for_team = (
await MCPRequestHandler._get_allowed_mcp_servers_for_team(user_api_key_auth)
await MCPRequestHandler._get_allowed_mcp_servers_for_team(
user_api_key_auth
)
)
#########################################################
@ -274,7 +309,9 @@ class MCPRequestHandler:
try:
key_object_permission = (
await prisma_client.db.litellm_objectpermissiontable.find_unique(
where={"object_permission_id": user_api_key_auth.object_permission_id},
where={
"object_permission_id": user_api_key_auth.object_permission_id
},
)
)
if key_object_permission is None:
@ -282,17 +319,21 @@ class MCPRequestHandler:
# Get direct MCP servers
direct_mcp_servers = key_object_permission.mcp_servers or []
# Get MCP servers from access groups
access_group_servers = await MCPRequestHandler._get_mcp_servers_from_access_groups(
key_object_permission.mcp_access_groups or []
access_group_servers = (
await MCPRequestHandler._get_mcp_servers_from_access_groups(
key_object_permission.mcp_access_groups or []
)
)
# Combine both lists
all_servers = direct_mcp_servers + access_group_servers
return list(set(all_servers))
except Exception as e:
verbose_logger.warning(f"Failed to get allowed MCP servers for key: {str(e)}")
verbose_logger.warning(
f"Failed to get allowed MCP servers for key: {str(e)}"
)
return []
@staticmethod
@ -318,10 +359,10 @@ class MCPRequestHandler:
return []
try:
team_obj: Optional[LiteLLM_TeamTable] = (
await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
team_obj: Optional[
LiteLLM_TeamTable
] = await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
if team_obj is None:
verbose_logger.debug("team_obj is None")
@ -333,21 +374,27 @@ class MCPRequestHandler:
# Get direct MCP servers
direct_mcp_servers = object_permissions.mcp_servers or []
# Get MCP servers from access groups
access_group_servers = await MCPRequestHandler._get_mcp_servers_from_access_groups(
object_permissions.mcp_access_groups or []
access_group_servers = (
await MCPRequestHandler._get_mcp_servers_from_access_groups(
object_permissions.mcp_access_groups or []
)
)
# Combine both lists
all_servers = direct_mcp_servers + access_group_servers
return list(set(all_servers))
except Exception as e:
verbose_logger.warning(f"Failed to get allowed MCP servers for team: {str(e)}")
verbose_logger.warning(
f"Failed to get allowed MCP servers for team: {str(e)}"
)
return []
@staticmethod
def _get_config_server_ids_for_access_groups(config_mcp_servers, access_groups: List[str]) -> Set[str]:
def _get_config_server_ids_for_access_groups(
config_mcp_servers, access_groups: List[str]
) -> Set[str]:
"""
Helper to get server_ids from config-loaded servers that match any of the given access groups.
"""
@ -359,7 +406,9 @@ class MCPRequestHandler:
return server_ids
@staticmethod
async def _get_db_server_ids_for_access_groups(prisma_client, access_groups: List[str]) -> Set[str]:
async def _get_db_server_ids_for_access_groups(
prisma_client, access_groups: List[str]
) -> Set[str]:
"""
Helper to get server_ids from DB servers that match any of the given access groups.
"""
@ -367,21 +416,19 @@ class MCPRequestHandler:
if access_groups and prisma_client is not None:
try:
mcp_servers = await prisma_client.db.litellm_mcpservertable.find_many(
where={
"mcp_access_groups": {
"hasSome": access_groups
}
}
where={"mcp_access_groups": {"hasSome": access_groups}}
)
for server in mcp_servers:
server_ids.add(server.server_id)
except Exception as e:
verbose_logger.debug(f"Error getting MCP servers from access groups: {e}")
verbose_logger.debug(
f"Error getting MCP servers from access groups: {e}"
)
return server_ids
@staticmethod
async def _get_mcp_servers_from_access_groups(
access_groups: List[str]
access_groups: List[str],
) -> List[str]:
"""
Resolve MCP access groups to server IDs by querying BOTH the MCP server table (DB) AND config-loaded servers
@ -390,22 +437,28 @@ class MCPRequestHandler:
try:
# Import here to avoid circular import
from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
# Use the new helper for config-loaded servers
server_ids = MCPRequestHandler._get_config_server_ids_for_access_groups(
global_mcp_server_manager.config_mcp_servers, access_groups
)
# Use the new helper for DB servers
db_server_ids = await MCPRequestHandler._get_db_server_ids_for_access_groups(
prisma_client, access_groups
db_server_ids = (
await MCPRequestHandler._get_db_server_ids_for_access_groups(
prisma_client, access_groups
)
)
server_ids.update(db_server_ids)
return list(server_ids)
except Exception as e:
verbose_logger.warning(f"Failed to get MCP servers from access groups: {str(e)}")
verbose_logger.warning(
f"Failed to get MCP servers from access groups: {str(e)}"
)
return []
@staticmethod
@ -418,8 +471,8 @@ class MCPRequestHandler:
from typing import List
access_groups: List[str] = []
access_groups_for_key = (
await MCPRequestHandler._get_mcp_access_groups_for_key(user_api_key_auth)
access_groups_for_key = await MCPRequestHandler._get_mcp_access_groups_for_key(
user_api_key_auth
)
access_groups_for_team = (
await MCPRequestHandler._get_mcp_access_groups_for_team(user_api_key_auth)
@ -482,10 +535,10 @@ class MCPRequestHandler:
verbose_logger.debug("prisma_client is None")
return []
team_obj: Optional[LiteLLM_TeamTable] = (
await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
team_obj: Optional[
LiteLLM_TeamTable
] = await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
if team_obj is None:
verbose_logger.debug("team_obj is None")
@ -502,10 +555,14 @@ class MCPRequestHandler:
"""
Extract and parse the x-mcp-access-groups header as a list of strings.
"""
mcp_access_groups_header = headers.get(MCPRequestHandler.LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME)
mcp_access_groups_header = headers.get(
MCPRequestHandler.LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME
)
if mcp_access_groups_header is not None:
try:
return [s.strip() for s in mcp_access_groups_header.split(",") if s.strip()]
return [
s.strip() for s in mcp_access_groups_header.split(",") if s.strip()
]
except Exception:
return None
return None
@ -516,4 +573,4 @@ class MCPRequestHandler:
Extract and parse the x-mcp-access-groups header from an ASGI scope.
"""
headers = MCPRequestHandler._safe_get_headers_from_scope(scope)
return MCPRequestHandler.get_mcp_access_groups_from_headers(headers)
return MCPRequestHandler.get_mcp_access_groups_from_headers(headers)

182
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -34,8 +34,6 @@ from litellm.proxy._experimental.mcp_server.utils import (
from litellm.proxy._types import (
LiteLLM_MCPServerTable,
MCPAuthType,
MCPSpecVersion,
MCPSpecVersionType,
MCPTransport,
MCPTransportType,
UserAPIKeyAuth,
@ -70,38 +68,6 @@ def _deserialize_env_dict(env_data: Any) -> Optional[Dict[str, str]]:
return env_data
def _convert_protocol_version_to_enum(
protocol_version: Optional[str | MCPSpecVersionType],
) -> MCPSpecVersionType:
"""
Convert string protocol version to MCPSpecVersion enum.
Args:
protocol_version: String protocol version, enum, or None
Returns:
MCPSpecVersionType: The enum value
"""
if not protocol_version:
return cast(MCPSpecVersionType, MCPSpecVersion.jun_2025)
# If it's already an MCPSpecVersion enum, return it
if isinstance(protocol_version, MCPSpecVersion):
return cast(MCPSpecVersionType, protocol_version)
# If it's a string, try to match it to enum values
if isinstance(protocol_version, str):
for version in MCPSpecVersion:
if version.value == protocol_version:
return cast(MCPSpecVersionType, version)
# If no match found, return default
verbose_logger.warning(
f"Unknown protocol version '{protocol_version}', using default"
)
return cast(MCPSpecVersionType, MCPSpecVersion.jun_2025)
class MCPServerManager:
def __init__(self):
self.registry: Dict[str, MCPServer] = {}
@ -113,8 +79,7 @@ class MCPServerManager:
"name": "zapier_mcp_server",
"url": "https://actions.zapier.com/mcp/sk-ak-2ew3bofIeQIkNoeKIdXrF1Hhhp/sse"
"transport": "sse",
"auth_type": "api_key",
"spec_version": "2025-03-26"
"auth_type": "api_key"
},
"uuid-2": {
"name": "google_drive_mcp_server",
@ -156,15 +121,13 @@ class MCPServerManager:
for server_name, server_config in mcp_servers_config.items():
validate_mcp_server_name(server_name)
_mcp_info: Dict[str, Any] = server_config.get("mcp_info", None) or {}
# Convert Dict[str, Any] to MCPInfo properly
mcp_info: MCPInfo = {
"server_name": _mcp_info.get("server_name", server_name),
"description": _mcp_info.get(
"description", server_config.get("description", None)
),
"logo_url": _mcp_info.get("logo_url", None),
"mcp_server_cost_info": _mcp_info.get("mcp_server_cost_info", None),
}
# Preserve all custom fields from config while setting defaults for core fields
mcp_info: MCPInfo = _mcp_info.copy()
# Set default values for core fields if not present
if "server_name" not in mcp_info:
mcp_info["server_name"] = server_name
if "description" not in mcp_info and server_config.get("description"):
mcp_info["description"] = server_config.get("description")
# Use alias for name if present, else server_name
alias = server_config.get("alias", None)
@ -223,7 +186,6 @@ class MCPServerManager:
server_name=server_name,
url=server_config.get("url", None) or "",
transport=server_config.get("transport", MCPTransport.http),
spec_version=server_config.get("spec_version", MCPSpecVersion.jun_2025),
auth_type=server_config.get("auth_type", None),
alias=alias,
)
@ -239,7 +201,6 @@ class MCPServerManager:
env=server_config.get("env", None) or {},
# TODO: utility fn the default values
transport=server_config.get("transport", MCPTransport.http),
spec_version=server_config.get("spec_version", MCPSpecVersion.jun_2025),
auth_type=server_config.get("auth_type", None),
authentication_token=server_config.get(
"authentication_token", server_config.get("auth_value", None)
@ -280,6 +241,14 @@ class MCPServerManager:
name_for_prefix = (
mcp_server.alias or mcp_server.server_name or mcp_server.server_id
)
# Preserve all custom fields from database while setting defaults for core fields
mcp_info: MCPInfo = _mcp_info.copy()
# Set default values for core fields if not present
if "server_name" not in mcp_info:
mcp_info["server_name"] = mcp_server.server_name or mcp_server.server_id
if "description" not in mcp_info and mcp_server.description:
mcp_info["description"] = mcp_server.description
new_server = MCPServer(
server_id=mcp_server.server_id,
name=name_for_prefix,
@ -287,13 +256,8 @@ class MCPServerManager:
server_name=getattr(mcp_server, "server_name", None),
url=mcp_server.url,
transport=cast(MCPTransportType, mcp_server.transport),
spec_version=_convert_protocol_version_to_enum(mcp_server.spec_version),
auth_type=cast(MCPAuthType, mcp_server.auth_type),
mcp_info=MCPInfo(
server_name=mcp_server.server_name or mcp_server.server_id,
description=mcp_server.description,
mcp_server_cost_info=_mcp_info.get("mcp_server_cost_info", None),
),
mcp_info=mcp_info,
# Stdio-specific fields
command=getattr(mcp_server, "command", None),
args=getattr(mcp_server, "args", None) or [],
@ -350,7 +314,6 @@ class MCPServerManager:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
) -> List[MCPTool]:
"""
List all tools available across all MCP Servers.
@ -390,7 +353,6 @@ class MCPServerManager:
tools = await self._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
mcp_protocol_version=mcp_protocol_version,
)
list_tools_result.extend(tools)
verbose_logger.info(
@ -414,7 +376,6 @@ class MCPServerManager:
self,
server: MCPServer,
mcp_auth_header: Optional[str] = None,
protocol_version: Optional[str] = None,
) -> MCPClient:
"""
Create an MCPClient instance for the given server.
@ -422,18 +383,12 @@ class MCPServerManager:
Args:
server (MCPServer): The server configuration
mcp_auth_header: MCP auth header to be passed to the MCP server. This is optional and will be used if provided.
protocol_version: Optional MCP protocol version to use. If not provided, uses server's default.
Returns:
MCPClient: Configured MCP client instance
"""
transport = server.transport or MCPTransport.sse
# Convert protocol version string to enum
protocol_version_enum = _convert_protocol_version_to_enum(
protocol_version or server.spec_version
)
# Handle stdio transport
if transport == MCPTransport.stdio:
# For stdio, we need to get the stdio config from the server
@ -450,7 +405,6 @@ class MCPServerManager:
auth_value=mcp_auth_header or server.authentication_token,
timeout=60.0,
stdio_config=stdio_config,
protocol_version=protocol_version_enum,
)
else:
# For HTTP/SSE transports
@ -461,14 +415,12 @@ class MCPServerManager:
auth_type=server.auth_type,
auth_value=mcp_auth_header or server.authentication_token,
timeout=60.0,
protocol_version=protocol_version_enum,
)
async def _get_tools_from_server(
self,
server: MCPServer,
mcp_auth_header: Optional[str] = None,
mcp_protocol_version: Optional[str] = None,
) -> List[MCPTool]:
"""
Helper method to get tools from a single MCP server with prefixed names.
@ -483,22 +435,18 @@ class MCPServerManager:
verbose_logger.debug(f"Connecting to url: {server.url}")
verbose_logger.info(f"_get_tools_from_server for {server.name}...")
protocol_version = (
mcp_protocol_version if mcp_protocol_version else server.spec_version
)
client = None
try:
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
protocol_version=protocol_version,
)
tools = await self._fetch_tools_with_timeout(client, server.name)
prefixed_tools = self._create_prefixed_tools(tools, server)
return prefixed_tools
except Exception as e:
@ -530,7 +478,7 @@ class MCPServerManager:
async def _list_tools_task():
try:
await client.connect()
tools = await client.list_tools()
verbose_logger.debug(f"Tools from {server_name}: {tools}")
return tools
@ -609,7 +557,6 @@ class MCPServerManager:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
proxy_logging_obj: Optional[ProxyLogging] = None,
) -> CallToolResult:
"""
@ -660,32 +607,54 @@ class MCPServerManager:
"arguments": arguments,
"server_name": server_name_from_prefix,
"user_api_key_auth": user_api_key_auth,
"user_api_key_user_id": getattr(user_api_key_auth, 'user_id', None) if user_api_key_auth else None,
"user_api_key_team_id": getattr(user_api_key_auth, 'team_id', None) if user_api_key_auth else None,
"user_api_key_end_user_id": getattr(user_api_key_auth, 'end_user_id', None) if user_api_key_auth else None,
"user_api_key_hash": getattr(user_api_key_auth, 'api_key_hash', None) if user_api_key_auth else None,
"user_api_key_user_id": getattr(user_api_key_auth, "user_id", None)
if user_api_key_auth
else None,
"user_api_key_team_id": getattr(user_api_key_auth, "team_id", None)
if user_api_key_auth
else None,
"user_api_key_end_user_id": getattr(
user_api_key_auth, "end_user_id", None
)
if user_api_key_auth
else None,
"user_api_key_hash": getattr(user_api_key_auth, "api_key_hash", None)
if user_api_key_auth
else None,
}
# Create MCP request object for processing
mcp_request_obj = proxy_logging_obj._create_mcp_request_object_from_kwargs(pre_hook_kwargs)
mcp_request_obj = proxy_logging_obj._create_mcp_request_object_from_kwargs(
pre_hook_kwargs
)
# Convert to LLM format for existing guardrail compatibility
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(mcp_request_obj, pre_hook_kwargs)
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(
mcp_request_obj, pre_hook_kwargs
)
try:
# Use standard pre_call_hook with call_type="mcp_call"
modified_data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_auth, #type: ignore
user_api_key_dict=user_api_key_auth, # type: ignore
data=synthetic_llm_data,
call_type="mcp_call" #type: ignore
call_type="mcp_call", # type: ignore
)
if modified_data:
# Convert response back to MCP format and apply modifications
modified_kwargs = proxy_logging_obj._convert_mcp_hook_response_to_kwargs(modified_data, pre_hook_kwargs)
modified_kwargs = (
proxy_logging_obj._convert_mcp_hook_response_to_kwargs(
modified_data, pre_hook_kwargs
)
)
if modified_kwargs.get("arguments") != arguments:
arguments = modified_kwargs["arguments"]
except (BlockedPiiEntityError, GuardrailRaisedException, HTTPException) as e:
except (
BlockedPiiEntityError,
GuardrailRaisedException,
HTTPException,
) as e:
# Re-raise guardrail exceptions to properly fail the MCP call
verbose_logger.error(
f"Guardrail blocked MCP tool call pre call: {str(e)}"
@ -706,11 +675,9 @@ class MCPServerManager:
client = self._create_mcp_client(
server=mcp_server,
mcp_auth_header=server_auth_header,
protocol_version=mcp_protocol_version,
)
async with client:
# Use the original tool name (without prefix) for the actual call
call_tool_params = MCPCallToolRequestParams(
name=original_tool_name,
@ -721,7 +688,7 @@ class MCPServerManager:
# Create synthetic LLM data for during hook processing
from litellm.types.llms.base import HiddenParams
from litellm.types.mcp import MCPDuringCallRequestObject
request_obj = MCPDuringCallRequestObject(
tool_name=name,
arguments=arguments,
@ -729,28 +696,29 @@ class MCPServerManager:
start_time=start_time.timestamp() if start_time else None,
hidden_params=HiddenParams(),
)
during_hook_kwargs = {
"name": name,
"arguments": arguments,
"server_name": server_name_from_prefix,
"user_api_key_auth": user_api_key_auth,
}
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(request_obj, during_hook_kwargs)
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(
request_obj, during_hook_kwargs
)
during_hook_task = asyncio.create_task(
proxy_logging_obj.during_call_hook(
user_api_key_dict=user_api_key_auth,
data=synthetic_llm_data,
call_type="mcp_call" #type: ignore
call_type="mcp_call", # type: ignore
)
)
tasks.append(during_hook_task)
tasks.append(asyncio.create_task(client.call_tool(call_tool_params)))
try:
mcp_responses = await asyncio.gather(*tasks)
# If proxy_logging_obj is None, the tool call result is at index 0
@ -839,19 +807,21 @@ class MCPServerManager:
)
verbose_logger.info("Loading MCP servers from database into registry...")
# perform authz check to filter the mcp servers user has access to
prisma_client = get_prisma_client_or_throw(
"Database not connected. Connect a database to your proxy"
)
db_mcp_servers = await get_all_mcp_servers(prisma_client)
verbose_logger.info(f"Found {len(db_mcp_servers)} MCP servers in database")
# ensure the global_mcp_server_manager is up to date with the db
for server in db_mcp_servers:
verbose_logger.debug(f"Adding server to registry: {server.server_id} ({server.server_name})")
verbose_logger.debug(
f"Adding server to registry: {server.server_id} ({server.server_name})"
)
self.add_update_server(server)
verbose_logger.info(f"Registry now contains {len(self.get_registry())} servers")
def get_mcp_server_by_id(self, server_id: str) -> Optional[MCPServer]:
@ -869,7 +839,6 @@ class MCPServerManager:
server_name: str,
url: str,
transport: str,
spec_version: str,
auth_type: Optional[str] = None,
alias: Optional[str] = None,
) -> str:
@ -885,7 +854,6 @@ class MCPServerManager:
server_name: Name of the server
url: Server URL
transport: Transport type (sse, http, etc.)
spec_version: MCP spec version
auth_type: Authentication type (optional)
alias: Server alias (optional)
@ -893,7 +861,9 @@ class MCPServerManager:
A deterministic server ID string
"""
# Create a string from all the identifying parameters
params_string = f"{server_name}|{url}|{transport}|{spec_version}|{auth_type or ''}|{alias or ''}"
params_string = (
f"{server_name}|{url}|{transport}|{auth_type or ''}|{alias or ''}"
)
# Generate SHA-256 hash
hash_object = hashlib.sha256(params_string.encode("utf-8"))
@ -1050,11 +1020,12 @@ class MCPServerManager:
alias=_server_config.alias,
url=_server_config.url,
transport=_server_config.transport,
spec_version=_server_config.spec_version,
auth_type=_server_config.auth_type,
created_at=datetime.datetime.now(),
updated_at=datetime.datetime.now(),
description=_server_config.mcp_info.get("description") if _server_config.mcp_info else None,
description=_server_config.mcp_info.get("description")
if _server_config.mcp_info
else None,
mcp_info=_server_config.mcp_info,
mcp_access_groups=_server_config.access_groups or [],
# Stdio-specific fields
@ -1111,7 +1082,6 @@ class MCPServerManager:
description=server.description,
url=server.url,
transport=server.transport,
spec_version=server.spec_version,
auth_type=server.auth_type,
created_at=server.created_at,
created_by=server.created_by,

128
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -23,7 +23,6 @@ router = APIRouter(
if MCP_AVAILABLE:
from litellm.experimental_mcp_client.client import MCPTool
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
_convert_protocol_version_to_enum,
global_mcp_server_manager,
)
from litellm.proxy._experimental.mcp_server.server import (
@ -34,18 +33,24 @@ if MCP_AVAILABLE:
########################################################
############ MCP Server REST API Routes #################
def _get_server_auth_header(
server, mcp_server_auth_headers: Optional[Dict[str, str]], mcp_auth_header: Optional[str]
server,
mcp_server_auth_headers: Optional[Dict[str, str]],
mcp_auth_header: Optional[str],
) -> Optional[str]:
"""Helper function to get server-specific auth header with case-insensitive matching."""
if mcp_server_auth_headers and server.alias:
normalized_server_alias = server.alias.lower()
normalized_headers = {k.lower(): v for k, v in mcp_server_auth_headers.items()}
normalized_headers = {
k.lower(): v for k, v in mcp_server_auth_headers.items()
}
server_auth = normalized_headers.get(normalized_server_alias)
if server_auth is not None:
return server_auth
elif mcp_server_auth_headers and server.server_name:
normalized_server_name = server.server_name.lower()
normalized_headers = {k.lower(): v for k, v in mcp_server_auth_headers.items()}
normalized_headers = {
k.lower(): v for k, v in mcp_server_auth_headers.items()
}
server_auth = normalized_headers.get(normalized_server_name)
if server_auth is not None:
return server_auth
@ -63,12 +68,11 @@ if MCP_AVAILABLE:
for tool in tools
]
async def _get_tools_for_single_server(server, server_auth_header, mcp_protocol_version):
async def _get_tools_for_single_server(server, server_auth_header):
"""Helper function to get tools for a single server."""
tools = await global_mcp_server_manager._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
mcp_protocol_version=mcp_protocol_version,
)
return _create_tool_response_objects(tools, server.mcp_info)
@ -104,17 +108,20 @@ if MCP_AVAILABLE:
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
try:
# Extract auth headers from request
headers = request.headers
mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(headers)
mcp_server_auth_headers = MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
mcp_protocol_version = headers.get(MCPRequestHandler.MCP_PROTOCOL_VERSION_HEADER_NAME)
mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers(
headers
)
mcp_server_auth_headers = (
MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
)
list_tools_result = []
error_message = None
# If server_id is specified, only query that specific server
if server_id:
server = global_mcp_server_manager.get_mcp_server_by_id(server_id)
@ -122,49 +129,67 @@ if MCP_AVAILABLE:
return {
"tools": [],
"error": "server_not_found",
"message": f"Server with id {server_id} not found"
"message": f"Server with id {server_id} not found",
}
server_auth_header = _get_server_auth_header(server, mcp_server_auth_headers, mcp_auth_header)
server_auth_header = _get_server_auth_header(
server, mcp_server_auth_headers, mcp_auth_header
)
try:
list_tools_result = await _get_tools_for_single_server(server, server_auth_header, mcp_protocol_version)
list_tools_result = await _get_tools_for_single_server(
server, server_auth_header
)
except Exception as e:
verbose_logger.exception(f"Error getting tools from {server.name}: {e}")
verbose_logger.exception(
f"Error getting tools from {server.name}: {e}"
)
return {
"tools": [],
"error": "server_error",
"message": f"Failed to get tools from server {server.name}: {str(e)}"
"message": f"Failed to get tools from server {server.name}: {str(e)}",
}
else:
# Query all servers
errors = []
for server in global_mcp_server_manager.get_registry().values():
server_auth_header = _get_server_auth_header(server, mcp_server_auth_headers, mcp_auth_header)
server_auth_header = _get_server_auth_header(
server, mcp_server_auth_headers, mcp_auth_header
)
try:
tools_result = await _get_tools_for_single_server(server, server_auth_header, mcp_protocol_version)
tools_result = await _get_tools_for_single_server(
server, server_auth_header
)
list_tools_result.extend(tools_result)
except Exception as e:
verbose_logger.exception(f"Error getting tools from {server.name}: {e}")
verbose_logger.exception(
f"Error getting tools from {server.name}: {e}"
)
errors.append(f"{server.name}: {str(e)}")
continue
if errors and not list_tools_result:
error_message = "Failed to get tools from servers: " + "; ".join(errors)
error_message = "Failed to get tools from servers: " + "; ".join(
errors
)
return {
"tools": list_tools_result,
"error": "partial_failure" if error_message else None,
"message": error_message if error_message else "Successfully retrieved tools"
"message": error_message
if error_message
else "Successfully retrieved tools",
}
except Exception as e:
verbose_logger.exception("Unexpected error in list_tool_rest_api: %s", str(e))
verbose_logger.exception(
"Unexpected error in list_tool_rest_api: %s", str(e)
)
return {
"tools": [],
"error": "unexpected_error",
"message": f"An unexpected error occurred: {str(e)}"
"message": f"An unexpected error occurred: {str(e)}",
}
@router.post("/tools/call", dependencies=[Depends(user_api_key_auth)])
@ -196,9 +221,9 @@ if MCP_AVAILABLE:
detail={
"error": "blocked_pii_entity",
"message": str(e),
"entity_type": getattr(e, 'entity_type', None),
"guardrail_name": getattr(e, 'guardrail_name', None)
}
"entity_type": getattr(e, "entity_type", None),
"guardrail_name": getattr(e, "guardrail_name", None),
},
)
except GuardrailRaisedException as e:
verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {str(e)}")
@ -207,8 +232,8 @@ if MCP_AVAILABLE:
detail={
"error": "guardrail_violation",
"message": str(e),
"guardrail_name": getattr(e, 'guardrail_name', None)
}
"guardrail_name": getattr(e, "guardrail_name", None),
},
)
except HTTPException as e:
# Re-raise HTTPException as-is to preserve status code and detail
@ -220,10 +245,10 @@ if MCP_AVAILABLE:
status_code=500,
detail={
"error": "internal_server_error",
"message": f"An unexpected error occurred: {str(e)}"
}
"message": f"An unexpected error occurred: {str(e)}",
},
)
########################################################
# MCP Connection testing routes
# /health -> Test if we can connect to the MCP server
@ -234,15 +259,15 @@ if MCP_AVAILABLE:
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
NewMCPServerRequest,
)
async def _execute_with_mcp_client(request: NewMCPServerRequest, operation):
"""
Common helper to create MCP client, execute operation, and ensure proper cleanup.
Args:
request: MCP server configuration
operation: Async function that takes a client and returns the operation result
Returns:
Operation result or error response
"""
@ -254,15 +279,14 @@ if MCP_AVAILABLE:
name=request.alias or request.server_name or "",
url=request.url,
transport=request.transport,
spec_version=_convert_protocol_version_to_enum(request.spec_version),
auth_type=request.auth_type,
mcp_info=request.mcp_info,
),
mcp_auth_header=None,
)
return await operation(client)
except Exception as e:
verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True)
return {"status": "error", "message": "An internal error has occurred."}
@ -273,6 +297,7 @@ if MCP_AVAILABLE:
await client.disconnect()
except Exception as e:
verbose_logger.warning(f"Error disconnecting MCP client: {e}")
@router.post("/test/connection")
async def test_connection(
request: NewMCPServerRequest,
@ -280,13 +305,13 @@ if MCP_AVAILABLE:
"""
Test if we can connect to the provided MCP server before adding it
"""
async def _test_connection_operation(client):
await client.connect()
return {"status": "ok"}
return await _execute_with_mcp_client(request, _test_connection_operation)
@router.post("/test/tools/list")
async def test_tools_list(
request: NewMCPServerRequest,
@ -295,13 +320,16 @@ if MCP_AVAILABLE:
"""
Preview tools available from MCP server before adding it
"""
async def _list_tools_operation(client):
list_tools_result: List[MCPTool] = await client.list_tools()
model_dumped_tools: List[dict] = [tool.model_dump() for tool in list_tools_result]
model_dumped_tools: List[dict] = [
tool.model_dump() for tool in list_tools_result
]
return {
"tools": model_dumped_tools,
"error": None,
"message": "Successfully retrieved tools"
"message": "Successfully retrieved tools",
}
return await _execute_with_mcp_client(request, _list_tools_operation)

242
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -130,7 +130,9 @@ if MCP_AVAILABLE:
await _sse_session_manager_cm.__aenter__()
_SESSION_MANAGERS_INITIALIZED = True
verbose_logger.info("MCP Server started with StreamableHTTP and SSE session managers!")
verbose_logger.info(
"MCP Server started with StreamableHTTP and SSE session managers!"
)
async def shutdown_session_managers():
"""Shutdown the session managers."""
@ -171,11 +173,18 @@ if MCP_AVAILABLE:
"""
try:
# Get user authentication from context variable
user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers, mcp_protocol_version = (
get_auth_context()
(
user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
) = get_auth_context()
verbose_logger.debug(
f"MCP list_tools - User API Key Auth from context: {user_api_key_auth}"
)
verbose_logger.debug(
f"MCP list_tools - MCP servers from context: {mcp_servers}"
)
verbose_logger.debug(f"MCP list_tools - User API Key Auth from context: {user_api_key_auth}")
verbose_logger.debug(f"MCP list_tools - MCP servers from context: {mcp_servers}")
verbose_logger.debug(
f"MCP list_tools - MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
)
@ -186,9 +195,10 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
verbose_logger.info(f"MCP list_tools - Successfully returned {len(tools)} tools")
verbose_logger.info(
f"MCP list_tools - Successfully returned {len(tools)} tools"
)
return tools
except Exception as e:
verbose_logger.exception(f"Error in list_tools endpoint: {str(e)}")
@ -220,9 +230,16 @@ if MCP_AVAILABLE:
from litellm.proxy.proxy_server import proxy_config
# Validate arguments
user_api_key_auth, mcp_auth_header, _, mcp_server_auth_headers, mcp_protocol_version = get_auth_context()
(
user_api_key_auth,
mcp_auth_header,
_,
mcp_server_auth_headers,
) = get_auth_context()
verbose_logger.debug(f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}")
verbose_logger.debug(
f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}"
)
try:
# Create a body date for logging
body_data = {"name": name, "arguments": arguments}
@ -249,17 +266,22 @@ if MCP_AVAILABLE:
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
**data, # for logging
)
except BlockedPiiEntityError as e:
verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {str(e)}")
# Return error as text content for MCP protocol
return [TextContent(text=f"Error: Blocked PII entity detected - {str(e)}", type="text")]
return [
TextContent(
text=f"Error: Blocked PII entity detected - {str(e)}", type="text"
)
]
except GuardrailRaisedException as e:
verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {str(e)}")
# Return error as text content for MCP protocol
return [TextContent(text=f"Error: Guardrail violation - {str(e)}", type="text")]
return [
TextContent(text=f"Error: Guardrail violation - {str(e)}", type="text")
]
except HTTPException as e:
verbose_logger.error(f"HTTPException in MCP tool call: {str(e)}")
# Return error as text content for MCP protocol
@ -287,6 +309,7 @@ if MCP_AVAILABLE:
Get the filtered MCP servers from the MCP server names
"""
from typing import Set
filtered_server_ids: Set[str] = set()
# Filter servers based on mcp_servers parameter if provided
if mcp_servers is not None:
@ -297,7 +320,11 @@ if MCP_AVAILABLE:
server = global_mcp_server_manager.get_mcp_server_by_id(server_id)
if server:
match_list = [s.lower() for s in [server.alias, server.server_name, server_id] if s is not None]
match_list = [
s.lower()
for s in [server.alias, server.server_name, server_id]
if s is not None
]
if server_or_group.lower() in match_list:
filtered_server_ids.add(server_id)
@ -306,19 +333,23 @@ if MCP_AVAILABLE:
if not server_name_matched:
try:
access_group_server_ids = await MCPRequestHandler._get_mcp_servers_from_access_groups(
[server_or_group]
access_group_server_ids = (
await MCPRequestHandler._get_mcp_servers_from_access_groups(
[server_or_group]
)
)
# Only include servers that the user has access to
for server_id in access_group_server_ids:
if server_id in allowed_mcp_servers:
filtered_server_ids.add(server_id)
except Exception as e:
verbose_logger.debug(f"Could not resolve '{server_or_group}' as access group: {e}")
verbose_logger.debug(
f"Could not resolve '{server_or_group}' as access group: {e}"
)
if filtered_server_ids:
allowed_mcp_servers = list(filtered_server_ids)
return allowed_mcp_servers
async def _get_tools_from_mcp_servers(
@ -326,7 +357,6 @@ if MCP_AVAILABLE:
mcp_auth_header: Optional[str],
mcp_servers: Optional[List[str]],
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
) -> List[MCPTool]:
"""
Helper method to fetch tools from MCP servers based on server filtering criteria.
@ -344,7 +374,9 @@ if MCP_AVAILABLE:
return []
# Get allowed MCP servers based on user permissions
allowed_mcp_servers = await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
allowed_mcp_servers = await global_mcp_server_manager.get_allowed_mcp_servers(
user_api_key_auth
)
if mcp_servers is not None:
allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names(
@ -352,7 +384,6 @@ if MCP_AVAILABLE:
allowed_mcp_servers=allowed_mcp_servers,
)
# Get tools from each allowed server
all_tools = []
for server_id in allowed_mcp_servers:
@ -375,15 +406,20 @@ if MCP_AVAILABLE:
tools = await global_mcp_server_manager._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
mcp_protocol_version=mcp_protocol_version,
)
all_tools.extend(tools)
verbose_logger.debug(f"Successfully fetched {len(tools)} tools from server {server.name}")
verbose_logger.debug(
f"Successfully fetched {len(tools)} tools from server {server.name}"
)
except Exception as e:
verbose_logger.exception(f"Error getting tools from server {server.name}: {str(e)}")
verbose_logger.exception(
f"Error getting tools from server {server.name}: {str(e)}"
)
# Continue with other servers instead of failing completely
verbose_logger.info(f"Successfully fetched {len(all_tools)} tools total from all MCP servers")
verbose_logger.info(
f"Successfully fetched {len(all_tools)} tools total from all MCP servers"
)
return all_tools
async def _list_mcp_tools(
@ -391,7 +427,6 @@ if MCP_AVAILABLE:
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
) -> List[MCPTool]:
"""
List all available MCP tools.
@ -415,11 +450,14 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
verbose_logger.debug(f"Successfully fetched {len(managed_tools)} tools from managed MCP servers")
verbose_logger.debug(
f"Successfully fetched {len(managed_tools)} tools from managed MCP servers"
)
except Exception as e:
verbose_logger.exception(f"Error getting tools from managed MCP servers: {str(e)}")
verbose_logger.exception(
f"Error getting tools from managed MCP servers: {str(e)}"
)
# Continue with empty managed tools list instead of failing completely
# Get tools from local registry
@ -430,10 +468,16 @@ if MCP_AVAILABLE:
# Convert local tools to MCPTool format
for tool in local_tools_raw:
# Convert from litellm.types.mcp_server.tool_registry.MCPTool to mcp.types.Tool
mcp_tool = MCPTool(name=tool.name, description=tool.description, inputSchema=tool.input_schema)
mcp_tool = MCPTool(
name=tool.name,
description=tool.description,
inputSchema=tool.input_schema,
)
local_tools.append(mcp_tool)
except Exception as e:
verbose_logger.exception(f"Error getting tools from local registry: {str(e)}")
verbose_logger.exception(
f"Error getting tools from local registry: {str(e)}"
)
# Continue with empty local tools list instead of failing completely
# Combine all tools
@ -448,7 +492,6 @@ if MCP_AVAILABLE:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
**kwargs: Any,
) -> List[Union[TextContent, ImageContent, EmbeddedResource]]:
"""
@ -456,35 +499,46 @@ if MCP_AVAILABLE:
"""
start_time = datetime.now()
if arguments is None:
raise HTTPException(status_code=400, detail="Request arguments are required")
raise HTTPException(
status_code=400, detail="Request arguments are required"
)
# Remove prefix from tool name for logging and processing
original_tool_name, server_name_from_prefix = get_server_name_prefix_tool_mcp(name)
standard_logging_mcp_tool_call: StandardLoggingMCPToolCall = _get_standard_logging_mcp_tool_call(
name=original_tool_name, # Use original name for logging
arguments=arguments,
server_name=server_name_from_prefix,
original_tool_name, server_name_from_prefix = get_server_name_prefix_tool_mcp(
name
)
standard_logging_mcp_tool_call: StandardLoggingMCPToolCall = (
_get_standard_logging_mcp_tool_call(
name=original_tool_name, # Use original name for logging
arguments=arguments,
server_name=server_name_from_prefix,
)
)
litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get(
"litellm_logging_obj", None
)
litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj", None)
if litellm_logging_obj:
litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = standard_logging_mcp_tool_call
litellm_logging_obj.model_call_details[
"mcp_tool_call_metadata"
] = standard_logging_mcp_tool_call
litellm_logging_obj.model = f"MCP: {name}"
# Try managed server tool first (pass the full prefixed name)
# Primary and recommended way to use MCP servers
#########################################################
mcp_server: Optional[MCPServer] = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
mcp_server: Optional[
MCPServer
] = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
if mcp_server:
standard_logging_mcp_tool_call["mcp_server_cost_info"] = (mcp_server.mcp_info or {}).get(
"mcp_server_cost_info"
)
standard_logging_mcp_tool_call["mcp_server_cost_info"] = (
mcp_server.mcp_info or {}
).get("mcp_server_cost_info")
response = await _handle_managed_mcp_tool(
name=name, # Pass the full name (potentially prefixed)
arguments=arguments,
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
litellm_logging_obj=litellm_logging_obj,
)
@ -537,7 +591,6 @@ if MCP_AVAILABLE:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
litellm_logging_obj: Optional[Any] = None,
) -> List[Union[TextContent, ImageContent, EmbeddedResource]]:
"""Handle tool execution for managed server tools"""
@ -577,42 +630,39 @@ if MCP_AVAILABLE:
Get the MCP servers from the path
"""
import re
mcp_servers_from_path: Optional[List[str]] = None
# Match /mcp/<servers>/<optional_path>
# Where <servers> can be comma-separated list of server names
# Match /mcp/<servers_and_maybe_path>
# Where servers can be comma-separated list of server names
# Server names can contain slashes (e.g., "custom_solutions/user_123")
mcp_path_match = re.match(r"^/mcp/([^?#]+?)(/[^?#]*)?(?:\?.*)?(?:#.*)?$", path)
mcp_path_match = re.match(r"^/mcp/([^?#]+)(?:\?.*)?(?:#.*)?$", path)
if mcp_path_match:
mcp_servers_str = mcp_path_match.group(1)
optional_path = mcp_path_match.group(2)
servers_and_path = mcp_path_match.group(1)
if mcp_servers_str:
# First, try to split by comma for comma-separated lists
if ',' in mcp_servers_str:
# For comma-separated lists, we need to handle the case where the last item
# might include the path (e.g., "zapier,group1/tools" -> ["zapier", "group1/tools"])
parts = [s.strip() for s in mcp_servers_str.split(",") if s.strip()]
# If there's an optional path AND the last part contains a slash that matches the optional path,
# remove the path portion from the last server name
if optional_path and len(parts) > 0 and '/' in parts[-1]:
last_part = parts[-1]
# Check if the last part ends with the optional path
if optional_path and last_part.endswith(optional_path.lstrip('/')):
# Remove the path portion from the last server name
parts[-1] = last_part[:-len(optional_path.lstrip('/'))]
mcp_servers_from_path = parts
if servers_and_path:
# Check if it contains commas (comma-separated servers)
if ',' in servers_and_path:
# For comma-separated, look for a path at the end
# Common patterns: /tools, /chat/completions, etc.
path_match = re.search(r'/([^/,]+(?:/[^/,]+)*)$', servers_and_path)
if path_match:
# Path found at the end, remove it from servers
path_part = '/' + path_match.group(1)
servers_part = servers_and_path[:-len(path_part)]
mcp_servers_from_path = [s.strip() for s in servers_part.split(',') if s.strip()]
else:
# No path, just comma-separated servers
mcp_servers_from_path = [s.strip() for s in servers_and_path.split(',') if s.strip()]
else:
# For single server, it might be just a name or contain slashes
# We need to determine where the server name ends and the path begins
# This is tricky - let's use the original logic but handle comma cases differently
single_server_match = re.match(r"^([^/]+(?:/[^/]+)?)(?:/.*)?$", mcp_servers_str)
# Single server case - use regex approach for server/path separation
# This handles cases like "custom_solutions/user_123/chat/completions"
# where we want to extract "custom_solutions/user_123" as the server name
single_server_match = re.match(r"^([^/]+(?:/[^/]+)?)(?:/.*)?$", servers_and_path)
if single_server_match:
server_name = single_server_match.group(1)
mcp_servers_from_path = [server_name]
else:
mcp_servers_from_path = [mcp_servers_str]
mcp_servers_from_path = [servers_and_path]
return mcp_servers_from_path
async def extract_mcp_auth_context(scope, path):
@ -627,7 +677,6 @@ if MCP_AVAILABLE:
mcp_auth_header,
_,
mcp_server_auth_headers,
mcp_protocol_version,
) = await MCPRequestHandler.process_mcp_request(scope)
mcp_servers = mcp_servers_from_path
else:
@ -636,11 +685,12 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
mcp_protocol_version,
) = await MCPRequestHandler.process_mcp_request(scope)
return user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers, mcp_protocol_version
return user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers
async def handle_streamable_http_mcp(scope: Scope, receive: Receive, send: Send) -> None:
async def handle_streamable_http_mcp(
scope: Scope, receive: Receive, send: Send
) -> None:
"""Handle MCP requests through StreamableHTTP."""
try:
path = scope.get("path", "")
@ -649,20 +699,19 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
mcp_protocol_version,
) = await extract_mcp_auth_context(scope, path)
verbose_logger.debug(f"MCP request mcp_servers (header/path): {mcp_servers}")
verbose_logger.debug(
f"MCP request mcp_servers (header/path): {mcp_servers}"
)
verbose_logger.debug(
f"MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
)
verbose_logger.debug(f"MCP protocol version: {mcp_protocol_version}")
# Set the auth context variable for easy access in MCP functions
set_auth_context(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
# Ensure session managers are initialized
@ -686,7 +735,9 @@ if MCP_AVAILABLE:
)
await error_response(scope, receive, send)
except Exception as response_error:
verbose_logger.exception(f"Failed to send error response: {response_error}")
verbose_logger.exception(
f"Failed to send error response: {response_error}"
)
# If we can't send a proper response, re-raise the original error
raise e
@ -699,19 +750,18 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
mcp_protocol_version,
) = await extract_mcp_auth_context(scope, path)
verbose_logger.debug(f"MCP request mcp_servers (header/path): {mcp_servers}")
verbose_logger.debug(
f"MCP request mcp_servers (header/path): {mcp_servers}"
)
verbose_logger.debug(
f"MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}"
)
verbose_logger.debug(f"MCP protocol version: {mcp_protocol_version}")
set_auth_context(
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
if not _SESSION_MANAGERS_INITIALIZED:
@ -733,7 +783,9 @@ if MCP_AVAILABLE:
)
await error_response(scope, receive, send)
except Exception as response_error:
verbose_logger.exception(f"Failed to send error response: {response_error}")
verbose_logger.exception(
f"Failed to send error response: {response_error}"
)
# If we can't send a proper response, re-raise the original error
raise e
@ -769,7 +821,6 @@ if MCP_AVAILABLE:
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
) -> None:
"""
Set the UserAPIKeyAuth in the auth context variable.
@ -785,13 +836,17 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
mcp_protocol_version=mcp_protocol_version,
)
auth_context_var.set(auth_user)
def get_auth_context() -> Tuple[
Optional[UserAPIKeyAuth], Optional[str], Optional[List[str]], Optional[Dict[str, str]], Optional[str]
]:
def get_auth_context() -> (
Tuple[
Optional[UserAPIKeyAuth],
Optional[str],
Optional[List[str]],
Optional[Dict[str, str]],
]
):
"""
Get the UserAPIKeyAuth from the auth context variable.
@ -806,9 +861,8 @@ if MCP_AVAILABLE:
auth_user.mcp_auth_header,
auth_user.mcp_servers,
auth_user.mcp_server_auth_headers,
auth_user.mcp_protocol_version,
)
return None, None, None, None, None
return None, None, None, None
########################################################
############ End of Auth Context Functions #############

0
litellm/proxy/_experimental/out/_next/static/fhuPj8WYsuMGymIUE7Xgu/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/0oPk2eYtSaTLaPyVixqA8/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/fhuPj8WYsuMGymIUE7Xgu/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/0oPk2eYtSaTLaPyVixqA8/_ssgManifest.js
View file

2
litellm/proxy/_experimental/out/_next/static/chunks/117-a0da667066d322b6.js → litellm/proxy/_experimental/out/_next/static/chunks/117-6a9f6a591033d4a0.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/154-6f752d9e0a5e497b.js → litellm/proxy/_experimental/out/_next/static/chunks/154-b1f2a106d0e0d77b.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/220-8af5927d18414264.js → litellm/proxy/_experimental/out/_next/static/chunks/220-89d73a525e307735.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/50-bb8a11a7610535aa.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/50-d0da2dd7acce2eb9.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/app/layout-df4939b4f2ec37d7.js Normal file
View file

@ -0,0 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{85210:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_1c856b', '__Inter_Fallback_1c856b'",fontStyle:"normal"},className:"__className_1c856b"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=85210)}),_N_E=n.O()}]);

1
litellm/proxy/_experimental/out/_next/static/chunks/app/layout-f4acf18888f0aa20.js
View file

@ -1 +0,0 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{96443:function(n,e,t){Promise.resolve().then(t.t.bind(t,39974,23)),Promise.resolve().then(t.t.bind(t,2778,23))},2778:function(){},39974:function(n){n.exports={style:{fontFamily:"'__Inter_b0dd8a', '__Inter_Fallback_b0dd8a'",fontStyle:"normal"},className:"__className_b0dd8a"}}},function(n){n.O(0,[919,986,971,117,744],function(){return n(n.s=96443)}),_N_E=n.O()}]);

2
litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-0dbadf20167b786c.js → litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub/page-13b00ef4a072d920.js
View file

@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{21024:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[50,521,154,162,971,117,744],function(){return e(e.s=21024)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[418],{67355:function(e,n,t){Promise.resolve().then(t.bind(t,52829))},52829:function(e,n,t){"use strict";t.r(n),t.d(n,{default:function(){return f}});var u=t(57437),s=t(2265),c=t(99376),r=t(72162);function f(){let e=(0,c.useSearchParams)().get("key"),[n,t]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&t(e)},[e]),(0,u.jsx)(r.Z,{accessToken:n})}}},function(e){e.O(0,[50,521,154,162,971,117,744],function(){return e(e.s=67355)}),_N_E=e.O()}]);

2
litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-f469bae327299fbb.js → litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-304b7041a3fa39f7.js
View file

@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{64563:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,172,971,117,744],function(){return e(e.s=64563)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{38520:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,172,971,117,744],function(){return e(e.s=38520)}),_N_E=e.O()}]);

2
litellm/proxy/_experimental/out/_next/static/chunks/app/onboarding/page-7828c2c64e97362a.js → litellm/proxy/_experimental/out/_next/static/chunks/app/onboarding/page-d0d85032bb87ba51.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/app/page-338773f18570e0d6.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/app/page-73b19c9fbf8cc64f.js Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/main-7e39698e9e999d78.js → litellm/proxy/_experimental/out/_next/static/chunks/main-ad370c92406567cc.js
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/main-app-4f7318ae681a6d94.js → litellm/proxy/_experimental/out/_next/static/chunks/main-app-bb35cb240fbb9dca.js
View file

@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{10264:function(e,n,t){Promise.resolve().then(t.t.bind(t,12846,23)),Promise.resolve().then(t.t.bind(t,19107,23)),Promise.resolve().then(t.t.bind(t,61060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,36423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(54278),n(10264)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{78483:function(e,n,t){Promise.resolve().then(t.t.bind(t,12846,23)),Promise.resolve().then(t.t.bind(t,19107,23)),Promise.resolve().then(t.t.bind(t,61060,23)),Promise.resolve().then(t.t.bind(t,4707,23)),Promise.resolve().then(t.t.bind(t,80,23)),Promise.resolve().then(t.t.bind(t,36423,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,117],function(){return n(54278),n(78483)}),_N_E=e.O()}]);

Some files were not shown because too many files have changed in this diff Show more