diff --git a/.circleci/config.yml b/.circleci/config.yml
index 4a442426976..0e53cfc0edb 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1521,6 +1521,7 @@ jobs:
- run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
- run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
- run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
+ - run: python ./tests/code_coverage_tests/check_fastuuid_usage.py
- run: helm lint ./deploy/charts/litellm-helm
db_migration_disable_update_check:
diff --git a/README.md b/README.md
index f74889fbb27..0918d2b1fa4 100644
--- a/README.md
+++ b/README.md
@@ -350,13 +350,21 @@ curl 'http://0.0.0.0:4000/key/generate' \
[**Read the Docs**](https://docs.litellm.ai/docs/)
-## Contributing
+## Run in Developer mode
+### Services
+1. Setup .env file in root
+2. Run dependant services `docker-compose up db prometheus`
-Interested in contributing? Contributions to LiteLLM Python SDK, Proxy Server, and LLM integrations are both accepted and highly encouraged!
+### Backend
+1. (In root) create virtual environment `python -m venv .venv`
+2. Activate virtual environment `source .venv/bin/activate`
+3. Install dependencies `pip install -e ".[all]"`
+4. Start proxy backend `python litellm/proxy_cli.py`
-**Quick start:** `git clone` → `make install-dev` → `make format` → `make lint` → `make test-unit`
-
-See our comprehensive [Contributing Guide (CONTRIBUTING.md)](CONTRIBUTING.md) for detailed instructions.
+### Frontend
+1. Navigate to `ui/litellm-dashboard`
+2. Install dependencies `npm install`
+3. Run `npm run dev` to start the dashboard
# Enterprise
For companies that need better security, user management and professional support
@@ -434,18 +442,3 @@ All these checks must pass before your PR can be merged.
-## Run in Developer mode
-### Services
-1. Setup .env file in root
-2. Run dependant services `docker-compose up db prometheus`
-
-### Backend
-1. (In root) create virtual environment `python -m venv .venv`
-2. Activate virtual environment `source .venv/bin/activate`
-3. Install dependencies `pip install -e ".[all]"`
-4. Start proxy backend `python3 /path/to/litellm/proxy_cli.py`
-
-### Frontend
-1. Navigate to `ui/litellm-dashboard`
-2. Install dependencies `npm install`
-3. Run `npm run dev` to start the dashboard
diff --git a/docs/my-website/docs/completion/shared_session.md b/docs/my-website/docs/completion/shared_session.md
new file mode 100644
index 00000000000..ff3da37f34f
--- /dev/null
+++ b/docs/my-website/docs/completion/shared_session.md
@@ -0,0 +1,213 @@
+# Shared Session Support
+
+## Overview
+
+LiteLLM now supports sharing `aiohttp.ClientSession` instances across multiple API calls to avoid creating unnecessary new sessions. This improves performance and resource utilization.
+
+## Usage
+
+### Basic Usage
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def main():
+ # Create a shared session
+ async with ClientSession() as shared_session:
+ # Use the same session for multiple calls
+ response1 = await acompletion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "Hello"}],
+ shared_session=shared_session
+ )
+
+ response2 = await acompletion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "How are you?"}],
+ shared_session=shared_session
+ )
+
+ # Both calls reuse the same session!
+
+asyncio.run(main())
+```
+
+### Without Shared Session (Default)
+
+```python
+import asyncio
+from litellm import acompletion
+
+async def main():
+ # Each call creates a new session
+ response1 = await acompletion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "Hello"}]
+ )
+
+ response2 = await acompletion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "How are you?"}]
+ )
+ # Two separate sessions created
+
+asyncio.run(main())
+```
+
+## Benefits
+
+- **Performance**: Reuse HTTP connections across multiple calls
+- **Resource Efficiency**: Reduce memory and connection overhead
+- **Better Control**: Manage session lifecycle explicitly
+- **Debugging**: Easy to trace which calls use which sessions
+
+## Debug Logging
+
+Enable debug logging to see session reuse in action:
+
+```python
+import os
+import litellm
+
+# Enable debug logging
+os.environ['LITELLM_LOG'] = 'DEBUG'
+
+# You'll see logs like:
+# 🔄 SHARED SESSION: acompletion called with shared_session (ID: 12345)
+# ✅ SHARED SESSION: Reusing existing ClientSession (ID: 12345)
+```
+
+## Common Patterns
+
+### FastAPI Integration
+
+```python
+from fastapi import FastAPI
+import aiohttp
+import litellm
+
+app = FastAPI()
+
+@app.post("/chat")
+async def chat(messages: list[dict]):
+ # Create session per request
+ async with aiohttp.ClientSession() as session:
+ return await litellm.acompletion(
+ model="gpt-4o",
+ messages=messages,
+ shared_session=session
+ )
+```
+
+### Batch Processing
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def process_batch(messages_list):
+ async with ClientSession() as shared_session:
+ tasks = []
+ for messages in messages_list:
+ task = acompletion(
+ model="gpt-4o",
+ messages=messages,
+ shared_session=shared_session
+ )
+ tasks.append(task)
+
+ # All tasks use the same session
+ results = await asyncio.gather(*tasks)
+ return results
+```
+
+### Custom Session Configuration
+
+```python
+import aiohttp
+import litellm
+
+# Create optimized session
+async with aiohttp.ClientSession(
+ timeout=aiohttp.ClientTimeout(total=180),
+ connector=aiohttp.TCPConnector(limit=300, limit_per_host=75)
+) as shared_session:
+
+ response = await litellm.acompletion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "Hello"}],
+ shared_session=shared_session
+ )
+```
+
+## Implementation Details
+
+The `shared_session` parameter is threaded through the entire LiteLLM call chain:
+
+1. **`acompletion()`** - Accepts `shared_session` parameter
+2. **`BaseLLMHTTPHandler`** - Passes session to HTTP client creation
+3. **`AsyncHTTPHandler`** - Uses existing session if provided
+4. **`LiteLLMAiohttpTransport`** - Reuses the session for HTTP requests
+
+## Backward Compatibility
+
+- **100% backward compatible** - Existing code works unchanged
+- **Optional parameter** - `shared_session=None` by default
+- **No breaking changes** - All existing functionality preserved
+
+## Testing
+
+Test the shared session functionality:
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def test_shared_session():
+ async with ClientSession() as session:
+ print(f"✅ Created session: {id(session)}")
+
+ try:
+ response = await acompletion(
+ model="gpt-4o",
+ messages=[{"role": "user", "content": "Hello"}],
+ shared_session=session,
+ api_key="your-api-key"
+ )
+ print(f"Response: {response.choices[0].message.content}")
+ except Exception as e:
+ print(f"✅ Expected error: {type(e).__name__}")
+
+ print("✅ Session control working!")
+
+asyncio.run(test_shared_session())
+```
+
+## Files Modified
+
+The shared session functionality was added to these files:
+
+- `litellm/main.py` - Added `shared_session` parameter to `acompletion()` and `completion()`
+- `litellm/llms/custom_httpx/http_handler.py` - Core session reuse logic
+- `litellm/llms/custom_httpx/llm_http_handler.py` - HTTP handler integration
+- `litellm/llms/openai/openai.py` - OpenAI provider integration
+- `litellm/llms/openai/common_utils.py` - OpenAI client creation
+- `litellm/llms/azure/chat/o_series_handler.py` - Azure O Series handler
+
+## Troubleshooting
+
+### Session Not Being Reused
+
+1. **Check debug logs**: Enable `LITELLM_LOG=DEBUG` to see session reuse messages
+2. **Verify session is not closed**: Ensure the session is still active when making calls
+3. **Check parameter passing**: Make sure `shared_session` is passed to all `acompletion()` calls
+
+### Performance Issues
+
+1. **Session configuration**: Tune `aiohttp.ClientSession` parameters for your use case
+2. **Connection limits**: Adjust `limit` and `limit_per_host` in `TCPConnector`
+3. **Timeout settings**: Configure appropriate timeouts for your environment
diff --git a/docs/my-website/docs/completion/usage.md b/docs/my-website/docs/completion/usage.md
index c388e5bfee1..2a9eab941ea 100644
--- a/docs/my-website/docs/completion/usage.md
+++ b/docs/my-website/docs/completion/usage.md
@@ -26,7 +26,6 @@ response = completion(
print(response.usage)
```
-> **Note:** LiteLLM supports endpoint bridging—if a model does not natively support a requested endpoint, LiteLLM will automatically route the call to the correct supported endpoint (such as bridging `/chat/completions` to `/responses` or vice versa) based on the model's `mode`set in `model_prices_and_context_window`.
## Streaming Usage
diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md
index cc3466fc103..9101d8e3751 100644
--- a/docs/my-website/docs/enterprise.md
+++ b/docs/my-website/docs/enterprise.md
@@ -1,11 +1,6 @@
import Image from '@theme/IdealImage';
# Enterprise
-
-:::info
-✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
-:::
-
For companies that need SSO, user management and professional support for LiteLLM Proxy
:::info
diff --git a/docs/my-website/docs/fine_tuning.md b/docs/my-website/docs/fine_tuning.md
index f3f955cb01d..f9a9297e062 100644
--- a/docs/my-website/docs/fine_tuning.md
+++ b/docs/my-website/docs/fine_tuning.md
@@ -13,8 +13,6 @@ This is an Enterprise only endpoint [Get Started with Enterprise here](https://c
| Feature | Supported | Notes |
|-------|-------|-------|
| Supported Providers | OpenAI, Azure OpenAI, Vertex AI | - |
-
-#### ⚡️See an exhaustive list of supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
| Cost Tracking | 🟡 | [Let us know if you need this](https://github.com/BerriAI/litellm/issues) |
| Logging | ✅ | Works across all logging integrations |
diff --git a/docs/my-website/docs/getting_started.md b/docs/my-website/docs/getting_started.md
index 6b2c1fd531e..15ee00a7273 100644
--- a/docs/my-website/docs/getting_started.md
+++ b/docs/my-website/docs/getting_started.md
@@ -32,8 +32,7 @@ Next Steps 👉 [Call all supported models - e.g. Claude-2, Llama2-70b, etc.](./
More details 👉
- [Completion() function details](./completion/)
-- [Overview of supported models / providers on LiteLLM](./providers/)
-- [Search all models / providers](https://models.litellm.ai/)
+- [All supported models / providers on LiteLLM](./providers/)
- [Build your own OpenAI proxy](https://github.com/BerriAI/liteLLM-proxy/tree/main)
## streaming
diff --git a/docs/my-website/docs/image_edits.md b/docs/my-website/docs/image_edits.md
index 84dddd5e4ad..246e1c70f0e 100644
--- a/docs/my-website/docs/image_edits.md
+++ b/docs/my-website/docs/image_edits.md
@@ -18,9 +18,6 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
| Supported LiteLLM Proxy Versions | 1.71.1+ | |
| Supported LLM providers | **OpenAI** | Currently only `openai` is supported |
- #### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
-
-
## Usage
### LiteLLM Python SDK
diff --git a/docs/my-website/docs/image_generation.md b/docs/my-website/docs/image_generation.md
index 8cd5803aa6c..7e7ff9922d6 100644
--- a/docs/my-website/docs/image_generation.md
+++ b/docs/my-website/docs/image_generation.md
@@ -279,8 +279,6 @@ print(f"response: {response}")
## Supported Providers
-#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
-
| Provider | Documentation Link |
|----------|-------------------|
| OpenAI | [OpenAI Image Generation →](./providers/openai) |
diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md
index 11d2963b7a3..3f5e1b479c3 100644
--- a/docs/my-website/docs/index.md
+++ b/docs/my-website/docs/index.md
@@ -524,15 +524,6 @@ try:
except OpenAIError as e:
print(e)
```
-### See How LiteLLM Transforms Your Requests
-
-Want to understand how LiteLLM parses and normalizes your LLM API requests? Use the `/utils/transform_request` endpoint to see exactly how your request is transformed internally.
-
-You can try it out now directly on our Demo App!
-Go to the [LiteLLM API docs for transform_request](https://litellm-api.up.railway.app/#/llm%20utils/transform_request_utils_transform_request_post)
-
-LiteLLM will show you the normalized, provider-agnostic version of your request. This is useful for debugging, learning, and understanding how LiteLLM handles different providers and options.
-
### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, Helicone, Promptlayer, Traceloop, Slack
diff --git a/docs/my-website/docs/integrations/index.md b/docs/my-website/docs/integrations/index.md
index 9731db6e751..95c922cce89 100644
--- a/docs/my-website/docs/integrations/index.md
+++ b/docs/my-website/docs/integrations/index.md
@@ -2,4 +2,17 @@
This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK).
+## AI Agent Frameworks
+- **[Letta](./letta.md)** - Build stateful LLM agents with persistent memory using LiteLLM Proxy
+
+## Development Tools
+- **[OpenWebUI](../tutorials/openweb_ui.md)** - Self-hosted ChatGPT-style interface
+
+## Observability & Monitoring
+- **[Langfuse](../observability/langfuse_integration.md)** - LLM observability and analytics
+- **[Prometheus](../proxy/prometheus.md)** - Metrics collection and monitoring
+- **[PagerDuty](../proxy/pagerduty.md)** - Incident response and alerting
+- **[Datadog](../observability/datadog.md)**
+
+
Click into each section to learn more about the integrations.
\ No newline at end of file
diff --git a/docs/my-website/docs/integrations/letta.md b/docs/my-website/docs/integrations/letta.md
new file mode 100644
index 00000000000..2afb82542f2
--- /dev/null
+++ b/docs/my-website/docs/integrations/letta.md
@@ -0,0 +1,928 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Letta Integration
+
+[Letta](https://github.com/letta-ai/letta) (formerly MemGPT) is a framework for building stateful LLM agents with persistent memory. This guide shows how to integrate both LiteLLM SDK and LiteLLM Proxy with Letta to leverage multiple LLM providers while building memory-enabled agents.
+
+## What is Letta?
+
+Letta allows you to build LLM agents that can:
+- Maintain long-term memory across conversations
+- Use function calling for tool interactions
+- Handle large context windows efficiently
+- Persist agent state and memory
+
+## Prerequisites
+
+```bash
+pip install letta litellm
+```
+
+## Quick Start
+
+
+
+
+### 1. Start LiteLLM Proxy
+
+First, create a configuration file for your LiteLLM proxy:
+
+```yaml
+# config.yaml
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: openai/gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+
+ - model_name: claude-3-sonnet
+ litellm_params:
+ model: anthropic/claude-3-sonnet-20240229
+ api_key: os.environ/ANTHROPIC_API_KEY
+
+ - model_name: gpt-3.5-turbo
+ litellm_params:
+ model: azure/gpt-35-turbo
+ api_key: os.environ/AZURE_API_KEY
+ api_base: os.environ/AZURE_API_BASE
+ api_version: "2023-07-01-preview"
+```
+
+Start the proxy:
+
+```bash
+litellm --config config.yaml --port 4000
+```
+
+### 2. Configure Letta with LiteLLM Proxy
+
+Configure Letta to use your LiteLLM proxy endpoint:
+
+```python
+import letta
+from letta import create_client
+
+# Configure Letta to use LiteLLM proxy
+client = create_client()
+
+# Configure the LLM endpoint
+client.set_default_llm_config(
+ model="gpt-4", # This should match a model from your LiteLLM config
+ model_endpoint_type="openai",
+ model_endpoint="http://localhost:4000", # Your LiteLLM proxy URL
+ context_window=8192
+)
+
+# Configure embedding endpoint (optional)
+client.set_default_embedding_config(
+ embedding_endpoint_type="openai",
+ embedding_endpoint="http://localhost:4000",
+ embedding_model="text-embedding-ada-002"
+)
+```
+
+
+
+
+### 1. Configure LiteLLM SDK
+
+Set up your API keys and configure LiteLLM:
+
+```python
+import os
+import litellm
+
+# Set your API keys
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+# Optional: Configure default settings
+litellm.set_verbose = True # For debugging
+```
+
+### 2. Create Custom LLM Wrapper for Letta
+
+Create a custom LLM wrapper that uses LiteLLM SDK:
+
+```python
+import letta
+from letta import create_client
+from letta.llm_api.llm_api_base import LLMConfig
+import litellm
+from typing import List, Dict, Any
+
+class LiteLLMWrapper:
+ def __init__(self, model: str):
+ self.model = model
+
+ def chat_completions_create(self, messages: List[Dict], **kwargs):
+ # Use LiteLLM SDK for completion
+ response = litellm.completion(
+ model=self.model,
+ messages=messages,
+ **kwargs
+ )
+ return response
+
+# Configure Letta with custom LiteLLM wrapper
+client = create_client()
+
+# Set up LLM configuration using direct SDK integration
+llm_config = LLMConfig(
+ model="gpt-4", # or "claude-3-sonnet", "azure/gpt-35-turbo", etc.
+ model_endpoint_type="openai",
+ context_window=8192
+)
+
+client.set_default_llm_config(llm_config)
+```
+
+
+
+
+### 3. Create and Use a Letta Agent
+
+
+
+
+```python
+import letta
+from letta import create_client
+
+# Create Letta client
+client = create_client()
+
+# Create a new agent
+agent_state = client.create_agent(
+ name="my-assistant",
+ system="You are a helpful assistant with persistent memory.",
+ llm_config=client.get_default_llm_config(),
+ embedding_config=client.get_default_embedding_config()
+)
+
+# Send a message to the agent
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="Hi! My name is Alice and I love reading science fiction books."
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+
+# Send another message - the agent will remember previous context
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="What did I tell you about my interests?"
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+```
+
+
+
+
+```python
+import letta
+from letta import create_client
+import litellm
+import os
+
+# Set up environment variables
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Create Letta client with LiteLLM integration
+client = create_client()
+
+# Create a new agent
+agent_state = client.create_agent(
+ name="my-assistant",
+ system="You are a helpful assistant with persistent memory.",
+ llm_config=client.get_default_llm_config(),
+ embedding_config=client.get_default_embedding_config()
+)
+
+# Send a message to the agent
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="Hi! My name is Alice and I love reading science fiction books."
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+
+# Send another message - the agent will remember previous context
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="What did I tell you about my interests?"
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+```
+
+
+
+
+## Advanced Configuration
+
+### Using Different Models for Different Agents
+
+
+
+
+```python
+from letta import LLMConfig, EmbeddingConfig
+
+# Create different LLM configurations pointing to your proxy
+gpt4_config = LLMConfig(
+ model="gpt-4",
+ model_endpoint_type="openai",
+ model_endpoint="http://localhost:4000",
+ context_window=8192
+)
+
+claude_config = LLMConfig(
+ model="claude-3-sonnet",
+ model_endpoint_type="openai", # Using OpenAI-compatible endpoint
+ model_endpoint="http://localhost:4000",
+ context_window=200000
+)
+
+# Create agents with different configurations
+research_agent = client.create_agent(
+ name="research-agent",
+ system="You are a research assistant specialized in analysis.",
+ llm_config=claude_config # Use Claude for research tasks
+)
+
+creative_agent = client.create_agent(
+ name="creative-agent",
+ system="You are a creative writing assistant.",
+ llm_config=gpt4_config # Use GPT-4 for creative tasks
+)
+```
+
+
+
+
+```python
+import os
+import litellm
+from letta import LLMConfig, EmbeddingConfig
+
+# Set up API keys for different providers
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+# Create different LLM configurations for direct SDK usage
+gpt4_config = LLMConfig(
+ model="openai/gpt-4", # Using LiteLLM model format
+ model_endpoint_type="openai",
+ context_window=8192
+)
+
+claude_config = LLMConfig(
+ model="anthropic/claude-3-sonnet-20240229", # Using LiteLLM model format
+ model_endpoint_type="openai",
+ context_window=200000
+)
+
+# Create agents with different configurations
+research_agent = client.create_agent(
+ name="research-agent",
+ system="You are a research assistant specialized in analysis.",
+ llm_config=claude_config # Use Claude for research tasks
+)
+
+creative_agent = client.create_agent(
+ name="creative-agent",
+ system="You are a creative writing assistant.",
+ llm_config=gpt4_config # Use GPT-4 for creative tasks
+)
+```
+
+
+
+
+### Function Calling with Tools
+
+
+
+
+```python
+# Define custom tools for your agent
+def search_web(query: str) -> str:
+ """Search the web for information"""
+ # Your web search implementation
+ return f"Search results for: {query}"
+
+def save_note(content: str) -> str:
+ """Save a note to persistent storage"""
+ # Your note saving implementation
+ return f"Note saved: {content}"
+
+# Create agent with tools (using proxy endpoint)
+agent_state = client.create_agent(
+ name="research-assistant",
+ system="You are a research assistant that can search the web and save notes.",
+ llm_config=client.get_default_llm_config(),
+ embedding_config=client.get_default_embedding_config(),
+ tools=[search_web, save_note]
+)
+
+# The agent can now use these tools
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="Search for recent developments in AI and save important findings."
+)
+```
+
+
+
+
+```python
+import litellm
+import os
+
+# Set up API keys
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Define custom tools for your agent
+def search_web(query: str) -> str:
+ """Search the web for information"""
+ # Your web search implementation
+ return f"Search results for: {query}"
+
+def save_note(content: str) -> str:
+ """Save a note to persistent storage"""
+ # Your note saving implementation
+ return f"Note saved: {content}"
+
+# Create agent with tools (using LiteLLM SDK directly)
+agent_state = client.create_agent(
+ name="research-assistant",
+ system="You are a research assistant that can search the web and save notes.",
+ llm_config=LLMConfig(
+ model="openai/gpt-4", # Direct model specification
+ model_endpoint_type="openai",
+ context_window=8192
+ ),
+ embedding_config=client.get_default_embedding_config(),
+ tools=[search_web, save_note]
+)
+
+# The agent can now use these tools
+response = client.user_message(
+ agent_id=agent_state.id,
+ message="Search for recent developments in AI and save important findings."
+)
+```
+
+
+
+
+## Authentication
+
+
+
+
+If your LiteLLM proxy requires authentication:
+
+```python
+import os
+from letta import LLMConfig
+
+# Set up authenticated configuration
+llm_config = LLMConfig(
+ model="gpt-4",
+ model_endpoint_type="openai",
+ model_endpoint="http://localhost:4000",
+ model_wrapper="openai",
+ context_window=8192
+)
+
+# If using API keys with your proxy
+os.environ["OPENAI_API_KEY"] = "your-litellm-proxy-api-key"
+
+client = create_client()
+client.set_default_llm_config(llm_config)
+```
+
+For proxy with authentication enabled:
+
+```yaml
+# config.yaml with auth
+general_settings:
+ master_key: "your-master-key"
+
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: openai/gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+```
+
+```python
+# Configure Letta with authenticated proxy
+llm_config = LLMConfig(
+ model="gpt-4",
+ model_endpoint_type="openai",
+ model_endpoint="http://localhost:4000",
+ context_window=8192,
+ api_key="your-master-key" # Proxy master key
+)
+```
+
+
+
+
+With LiteLLM SDK, set up your provider API keys directly:
+
+```python
+import os
+import litellm
+
+# Set up API keys for different providers
+os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-api-key"
+os.environ["AZURE_API_KEY"] = "your-azure-api-key"
+os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com"
+os.environ["AZURE_API_VERSION"] = "2023-07-01-preview"
+
+# Optional: Configure default settings
+litellm.api_key = os.environ.get("OPENAI_API_KEY") # Default key
+litellm.set_verbose = True # For debugging
+
+# Use in Letta configuration
+from letta import LLMConfig
+
+llm_config = LLMConfig(
+ model="openai/gpt-4", # Will use OPENAI_API_KEY automatically
+ model_endpoint_type="openai",
+ context_window=8192
+)
+
+# Or for Azure
+azure_config = LLMConfig(
+ model="azure/gpt-35-turbo",
+ model_endpoint_type="openai",
+ context_window=4096
+)
+```
+
+
+
+
+## Load Balancing and Fallbacks
+
+
+
+
+LiteLLM proxy's load balancing and fallback features work seamlessly with Letta:
+
+```yaml
+# config.yaml with fallbacks
+model_list:
+ - model_name: gpt-4
+ litellm_params:
+ model: openai/gpt-4
+ api_key: os.environ/OPENAI_API_KEY
+ tpm: 40000
+ rpm: 500
+
+ - model_name: gpt-4 # Same model name for fallback
+ litellm_params:
+ model: azure/gpt-4
+ api_key: os.environ/AZURE_API_KEY
+ api_base: os.environ/AZURE_API_BASE
+ api_version: "2023-07-01-preview"
+ tpm: 80000
+ rpm: 800
+
+router_settings:
+ routing_strategy: "usage-based-routing"
+ fallbacks: [{"gpt-4": ["azure/gpt-4"]}]
+```
+
+The proxy handles all routing, load balancing, and fallbacks transparently for Letta.
+
+
+
+
+With LiteLLM SDK, you can set up routing and fallbacks programmatically:
+
+```python
+import litellm
+from litellm import Router
+
+# Configure router with multiple models
+router = Router(
+ model_list=[
+ {
+ "model_name": "gpt-4",
+ "litellm_params": {
+ "model": "openai/gpt-4",
+ "api_key": os.environ["OPENAI_API_KEY"]
+ },
+ "tpm": 40000,
+ "rpm": 500
+ },
+ {
+ "model_name": "gpt-4", # Same name for fallback
+ "litellm_params": {
+ "model": "azure/gpt-4",
+ "api_key": os.environ["AZURE_API_KEY"],
+ "api_base": os.environ["AZURE_API_BASE"],
+ "api_version": "2023-07-01-preview"
+ },
+ "tpm": 80000,
+ "rpm": 800
+ }
+ ],
+ fallbacks=[{"gpt-4": ["azure/gpt-4"]}],
+ routing_strategy="usage-based-routing"
+)
+
+# Create custom completion function for Letta
+def custom_completion(messages, model="gpt-4", **kwargs):
+ return router.completion(
+ model=model,
+ messages=messages,
+ **kwargs
+ )
+
+# Use with Letta by monkey-patching or custom wrapper
+litellm.completion = custom_completion
+```
+
+
+
+
+## Monitoring and Observability
+
+
+
+
+Enable logging to track your Letta agents' LLM usage through the proxy:
+
+```yaml
+# config.yaml with logging
+model_list:
+ # ... your models
+
+litellm_settings:
+ success_callback: ["langfuse"] # or other observability tools
+
+environment_variables:
+ LANGFUSE_PUBLIC_KEY: "your-key"
+ LANGFUSE_SECRET_KEY: "your-secret"
+```
+
+View metrics in the proxy dashboard:
+```bash
+# Start proxy with UI
+litellm --config config.yaml --port 4000 --detailed_debug
+```
+
+
+
+
+Set up observability directly in your SDK integration:
+
+```python
+import litellm
+import os
+
+# Configure observability callbacks
+os.environ["LANGFUSE_PUBLIC_KEY"] = "your-key"
+os.environ["LANGFUSE_SECRET_KEY"] = "your-secret"
+
+# Set global callbacks
+litellm.success_callback = ["langfuse"]
+litellm.failure_callback = ["langfuse"]
+
+# Optional: Set up custom logging
+litellm.set_verbose = True
+
+# Create custom completion wrapper with logging
+def logged_completion(messages, model="gpt-4", **kwargs):
+ try:
+ response = litellm.completion(
+ model=model,
+ messages=messages,
+ **kwargs
+ )
+ # Custom logging logic here if needed
+ return response
+ except Exception as e:
+ # Custom error handling
+ print(f"LLM call failed: {e}")
+ raise
+
+# Use in Letta configuration
+litellm.completion = logged_completion
+```
+
+
+
+
+## Example: Multi-Agent System
+
+
+
+
+```python
+import letta
+from letta import create_client, LLMConfig
+
+client = create_client()
+
+# Create specialized agents using proxy endpoints
+agents = {}
+
+# Research agent using Claude for analysis
+agents['researcher'] = client.create_agent(
+ name="researcher",
+ system="You are a research specialist. Analyze information thoroughly.",
+ llm_config=LLMConfig(
+ model="claude-3-sonnet",
+ model_endpoint="http://localhost:4000",
+ model_endpoint_type="openai"
+ )
+)
+
+# Writer agent using GPT-4 for content creation
+agents['writer'] = client.create_agent(
+ name="writer",
+ system="You are a content writer. Create engaging, well-structured content.",
+ llm_config=LLMConfig(
+ model="gpt-4",
+ model_endpoint="http://localhost:4000",
+ model_endpoint_type="openai"
+ )
+)
+
+# Coordinator workflow
+def research_and_write_workflow(topic: str):
+ # Research phase
+ research_response = client.user_message(
+ agent_id=agents['researcher'].id,
+ message=f"Research the topic: {topic}. Provide key insights and data."
+ )
+
+ research_results = research_response.messages[-1].text
+
+ # Writing phase
+ write_response = client.user_message(
+ agent_id=agents['writer'].id,
+ message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
+ )
+
+ return write_response.messages[-1].text
+
+# Execute workflow
+article = research_and_write_workflow("The future of AI in healthcare")
+print(article)
+```
+
+
+
+
+```python
+import letta
+from letta import create_client, LLMConfig
+import litellm
+import os
+
+# Set up environment
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+client = create_client()
+
+# Create specialized agents using direct SDK models
+agents = {}
+
+# Research agent using Claude for analysis
+agents['researcher'] = client.create_agent(
+ name="researcher",
+ system="You are a research specialist. Analyze information thoroughly.",
+ llm_config=LLMConfig(
+ model="anthropic/claude-3-sonnet-20240229",
+ model_endpoint_type="openai"
+ )
+)
+
+# Writer agent using GPT-4 for content creation
+agents['writer'] = client.create_agent(
+ name="writer",
+ system="You are a content writer. Create engaging, well-structured content.",
+ llm_config=LLMConfig(
+ model="openai/gpt-4",
+ model_endpoint_type="openai"
+ )
+)
+
+# Cost-conscious agent using GPT-3.5
+agents['reviewer'] = client.create_agent(
+ name="reviewer",
+ system="You are an editor. Review and improve content quality.",
+ llm_config=LLMConfig(
+ model="openai/gpt-3.5-turbo",
+ model_endpoint_type="openai"
+ )
+)
+
+# Enhanced workflow with multiple agents
+def enhanced_workflow(topic: str):
+ # Research phase
+ research_response = client.user_message(
+ agent_id=agents['researcher'].id,
+ message=f"Research the topic: {topic}. Provide key insights and data."
+ )
+
+ research_results = research_response.messages[-1].text
+
+ # Writing phase
+ write_response = client.user_message(
+ agent_id=agents['writer'].id,
+ message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
+ )
+
+ draft_article = write_response.messages[-1].text
+
+ # Review phase
+ review_response = client.user_message(
+ agent_id=agents['reviewer'].id,
+ message=f"Please review and improve this article:\n\n{draft_article}"
+ )
+
+ return review_response.messages[-1].text
+
+# Execute enhanced workflow
+article = enhanced_workflow("The future of AI in healthcare")
+print(article)
+```
+
+
+
+
+## Best Practices
+
+
+
+
+1. **Model Selection**: Use appropriate models for different tasks:
+ - Claude for analysis and reasoning
+ - GPT-4 for creative tasks
+ - GPT-3.5-turbo for simple interactions
+
+2. **Proxy Configuration**:
+ - Set appropriate rate limits and timeouts
+ - Use fallbacks for reliability
+ - Enable authentication for production
+
+3. **Memory Management**: Letta handles memory automatically, but monitor usage with large contexts
+
+4. **Cost Optimization**:
+ - Use the proxy's budgeting features to control costs
+ - Set up rate limiting per user/team
+ - Monitor token usage through proxy dashboard
+
+5. **Monitoring**: Enable observability to track agent performance and token usage
+
+
+
+
+1. **Model Selection**: Choose models based on task requirements:
+ - Use `openai/gpt-4` for complex reasoning
+ - Use `anthropic/claude-3-sonnet-20240229` for analysis
+ - Use `openai/gpt-3.5-turbo` for cost-effective simple tasks
+
+2. **Error Handling**: Implement robust error handling with retries:
+ ```python
+ import litellm
+ from litellm import completion
+
+ # Set up retry logic
+ litellm.num_retries = 3
+ litellm.request_timeout = 60
+
+ # Custom error handling
+ def safe_completion(**kwargs):
+ try:
+ return completion(**kwargs)
+ except Exception as e:
+ print(f"LLM call failed: {e}")
+ # Implement fallback logic
+ return completion(model="openai/gpt-3.5-turbo", **kwargs)
+ ```
+
+3. **Cost Management**:
+ - Use cheaper models for non-critical tasks
+ - Implement token counting and budgets
+ - Cache responses when appropriate
+
+4. **Performance**:
+ - Use async operations for concurrent requests
+ - Implement connection pooling
+ - Monitor response times
+
+5. **Security**:
+ - Store API keys securely (environment variables)
+ - Rotate keys regularly
+ - Implement rate limiting
+
+
+
+
+## Troubleshooting
+
+
+
+
+### Connection Issues
+```bash
+# Test your LiteLLM proxy
+curl -X POST http://localhost:4000/v1/chat/completions \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "gpt-4",
+ "messages": [{"role": "user", "content": "Hello"}]
+ }'
+```
+
+### Configuration Debugging
+```python
+# Enable verbose logging
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+# Test Letta configuration
+client = create_client()
+print(client.get_default_llm_config())
+```
+
+### Common Proxy Issues
+- **Port conflicts**: Make sure port 4000 isn't in use
+- **Model not found**: Verify model names match your config.yaml
+- **Authentication errors**: Check master key configuration
+- **Rate limiting**: Monitor proxy logs for rate limit hits
+
+
+
+
+### API Key Issues
+```python
+import os
+import litellm
+
+# Check if API keys are set
+print("OpenAI Key:", os.environ.get("OPENAI_API_KEY", "Not set"))
+print("Anthropic Key:", os.environ.get("ANTHROPIC_API_KEY", "Not set"))
+
+# Test direct LiteLLM call
+try:
+ response = litellm.completion(
+ model="openai/gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hello"}]
+ )
+ print("LiteLLM working:", response.choices[0].message.content)
+except Exception as e:
+ print("LiteLLM error:", e)
+```
+
+### Configuration Debugging
+```python
+# Enable verbose logging
+litellm.set_verbose = True
+
+# Test model availability
+models = ["openai/gpt-4", "anthropic/claude-3-sonnet-20240229"]
+for model in models:
+ try:
+ response = litellm.completion(
+ model=model,
+ messages=[{"role": "user", "content": "Test"}],
+ max_tokens=10
+ )
+ print(f"✓ {model} working")
+ except Exception as e:
+ print(f"✗ {model} failed: {e}")
+```
+
+### Common SDK Issues
+- **Import errors**: Ensure `pip install litellm letta` is run
+- **Model format**: Use `provider/model` format (e.g., `openai/gpt-4`)
+- **API key format**: Different providers have different key formats
+- **Rate limits**: Implement exponential backoff for retries
+
+
+
+
+## Resources
+
+- [Letta Documentation](https://docs.letta.ai/)
+- [LiteLLM Proxy Documentation](../proxy/quick_start.md)
+- [LiteLLM SDK Documentation](../completion/input.md)
+- [Function Calling Guide](../completion/function_call.md)
+- [Observability Setup](../observability/langfuse_integration.md)
+- [Router Configuration](../routing.md)
\ No newline at end of file
diff --git a/docs/my-website/docs/moderation.md b/docs/my-website/docs/moderation.md
index f9c2810bc8a..95fe8b2856d 100644
--- a/docs/my-website/docs/moderation.md
+++ b/docs/my-website/docs/moderation.md
@@ -130,8 +130,6 @@ Here's the exact json output and type you can expect from all moderation calls:
## **Supported Providers**
-#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
-
| Provider |
|-------------|
| OpenAI |
diff --git a/docs/my-website/docs/observability/callbacks.md b/docs/my-website/docs/observability/callbacks.md
index b752bdc2764..040d83697d3 100644
--- a/docs/my-website/docs/observability/callbacks.md
+++ b/docs/my-website/docs/observability/callbacks.md
@@ -5,15 +5,13 @@
liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses.
:::tip
-**New to LiteLLM Callbacks?**
-
-- For proxy/server logging and observability, see the [Proxy Logging Guide](https://docs.litellm.ai/docs/proxy/logging).
-- To write your own callback logic, see the [Custom Callbacks Guide](https://docs.litellm.ai/docs/observability/custom_callback).
+**New to LiteLLM Callbacks?** Check out our comprehensive [Callback Management Guide](./callback_management.md) to understand when to use different callback hooks like `async_log_success_event` vs `async_post_call_success_hook`.
:::
+liteLLM supports:
-### Supported Callback Integrations
-
+- [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback)
+- [Callback Management Guide](./callback_management.md) - **Comprehensive guide for choosing the right hooks**
- [Lunary](https://lunary.ai/docs)
- [Langfuse](https://langfuse.com/docs)
- [LangSmith](https://www.langchain.com/langsmith)
@@ -23,20 +21,9 @@ liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`,
- [Sentry](https://docs.sentry.io/platforms/python/)
- [PostHog](https://posthog.com/docs/libraries/python)
- [Slack](https://slack.dev/bolt-python/concepts)
-- [Arize](https://docs.arize.com/)
-- [PromptLayer](https://docs.promptlayer.com/)
This is **not** an extensive list. Please check the dropdown for all logging integrations.
-### Related Cookbooks
-Try out our cookbooks for code snippets and interactive demos:
-
-- [Langfuse Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Langfuse.ipynb)
-- [Lunary Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Lunary.ipynb)
-- [Arize Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Arize.ipynb)
-- [Proxy + Langfuse Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb)
-- [PromptLayer Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/LiteLLM_PromptLayer.ipynb)
-
### Quick Start
```python
diff --git a/docs/my-website/docs/observability/custom_callback.md b/docs/my-website/docs/observability/custom_callback.md
index cfe97ca42c0..c206c23d0f4 100644
--- a/docs/my-website/docs/observability/custom_callback.md
+++ b/docs/my-website/docs/observability/custom_callback.md
@@ -67,23 +67,6 @@ asyncio.run(completion())
- `async_post_call_success_hook` - Access user data + modify responses
- `async_pre_call_hook` - Modify requests before sending
-### Example: Modifying the Response in async_post_call_success_hook
-
-You can use `async_post_call_success_hook` to add custom headers or metadata to the response before it is returned to the client. For example:
-
-```python
-async def async_post_call_success_hook(data, user_api_key_dict, response):
- # Add a custom header to the response
- additional_headers = getattr(response, "_hidden_params", {}).get("additional_headers", {}) or {}
- additional_headers["x-litellm-custom-header"] = "my-value"
- if not hasattr(response, "_hidden_params"):
- response._hidden_params = {}
- response._hidden_params["additional_headers"] = additional_headers
- return response
-```
-
-This allows you to inject custom metadata or headers into the response for downstream consumers. You can use this pattern to pass information to clients, proxies, or observability tools.
-
## Callback Functions
If you just want to log on a specific event (e.g. on input) - you can use callback functions.
diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md
index fe996099145..86e9ac5e3e6 100644
--- a/docs/my-website/docs/providers/bedrock.md
+++ b/docs/my-website/docs/providers/bedrock.md
@@ -2340,39 +2340,6 @@ response = completion(
Make the bedrock completion call
----
-
-### Required AWS IAM Policy for AssumeRole
-
-To use `aws_role_name` (STS AssumeRole) with LiteLLM, your IAM user or role **must** have permission to call `sts:AssumeRole` on the target role. If you see an error like:
-
-```
-An error occurred (AccessDenied) when calling the AssumeRole operation: User: arn:aws:sts::...:assumed-role/litellm-ecs-task-role/... is not authorized to perform: sts:AssumeRole on resource: arn:aws:iam::...:role/Enterprise/BedrockCrossAccountConsumer
-```
-
-This means the IAM identity running LiteLLM does **not** have permission to assume the target role. You must update your IAM policy to allow this action.
-
-#### Example IAM Policy
-
-Replace `` with the ARN of the role you want to assume (e.g., `arn:aws:iam::123456789012:role/Enterprise/BedrockCrossAccountConsumer`).
-
-```json
-{
- "Version": "2012-10-17",
- "Statement": [
- {
- "Effect": "Allow",
- "Action": "sts:AssumeRole",
- "Resource": ""
- }
- ]
-}
-```
-
-**Note:** The target role itself must also trust the calling IAM identity (via its trust policy) for AssumeRole to succeed. See [AWS AssumeRole docs](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-api.html) for more details.
-
----
-
diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md
index 260cc55c2e9..3f4d1068958 100644
--- a/docs/my-website/docs/providers/vertex.md
+++ b/docs/my-website/docs/providers/vertex.md
@@ -196,19 +196,7 @@ model_list:
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json" # [OPTIONAL] Do this OR `!gcloud auth application-default login` - run this to add vertex credentials to your env
```
-or
-```yaml
-model_list:
- - model_name: gemini-pro
- litellm_params:
- model: vertex_ai/gemini-1.5-pro
- litellm_credential_name: vertex-global
- vertex_project: project-name-here
- vertex_location: global
- base_model: gemini
- model_info:
- provider: Vertex
-```
+
2. Start Proxy
```
@@ -827,6 +815,77 @@ Use Vertex AI context caching is supported by calling provider api directly. (Un
[**Go straight to provider**](../pass_through/vertex_ai.md#context-caching)
+#### 1. Create the Cache
+
+First, create the cache by sending a `POST` request to the `cachedContents` endpoint via the LiteLLM proxy.
+
+
+
+
+```bash
+curl http://0.0.0.0:4000/vertex_ai/v1/projects/{project_id}/locations/{location}/cachedContents \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer $LITELLM_KEY" \
+ -d '{
+ "model": "projects/{project_id}/locations/{location}/publishers/google/models/gemini-2.5-flash",
+ "displayName": "example_cache",
+ "contents": [{
+ "role": "user",
+ "parts": [{
+ "text": ".... a long book to be cached"
+ }]
+ }]
+ }'
+```
+
+
+
+
+#### 2. Get the Cache Name from the Response
+
+Vertex AI will return a response containing the `name` of the cached content. This name is the identifier for your cached data.
+
+```json
+{
+ "name": "projects/12341234/locations/{location}/cachedContents/123123123123123",
+ "model": "projects/{project_id}/locations/{location}/publishers/google/models/gemini-2.5-flash",
+ "createTime": "2025-09-23T19:13:50.674976Z",
+ "updateTime": "2025-09-23T19:13:50.674976Z",
+ "expireTime": "2025-09-23T20:13:50.655988Z",
+ "displayName": "example_cache",
+ "usageMetadata": {
+ "totalTokenCount": 1246,
+ "textCount": 5132
+ }
+}
+```
+
+#### 3. Use the Cached Content
+
+Use the `name` from the response as `cachedContent` or `cached_content` in subsequent API calls to reuse the cached information. This is passed in the body of your request to `/chat/completions`.
+
+
+
+
+```bash
+
+curl http://0.0.0.0:4000/chat/completions \
+ -H "Content-Type: application/json" \
+ -H "Authorization: Bearer $LITELLM_KEY" \
+ -d '{
+ "cachedContent": "projects/545201925769/locations/us-central1/cachedContents/4511135542628319232",
+ "model": "gemini-2.5-flash",
+ "messages": [
+ {
+ "role": "user",
+ "content": "what is the book about?"
+ }
+ ]
+ }'
+```
+
+
+
## Pre-requisites
* `pip install google-cloud-aiplatform` (pre-installed on proxy docker image)
@@ -2736,7 +2795,3 @@ Once that's done, when you deploy the new container in the Google Cloud Run serv
s/o @[Darien Kindlund](https://www.linkedin.com/in/kindlund/) for this tutorial
-
-
-
-
diff --git a/docs/my-website/docs/proxy/caching.md b/docs/my-website/docs/proxy/caching.md
index 617609cf08a..1fb7385f689 100644
--- a/docs/my-website/docs/proxy/caching.md
+++ b/docs/my-website/docs/proxy/caching.md
@@ -958,19 +958,6 @@ curl http://localhost:4000/v1/chat/completions \
-
-## Redis max_connections
-
-You can set the `max_connections` parameter in your `cache_params` for Redis. This is passed directly to the Redis client and controls the maximum number of simultaneous connections in the pool. If you see errors like `No connection available`, try increasing this value:
-
-```yaml
-litellm_settings:
- cache: true
- cache_params:
- type: redis
- max_connections: 100
-```
-
## Supported `cache_params` on proxy config.yaml
```yaml
@@ -979,7 +966,6 @@ cache_params:
ttl: Optional[float]
default_in_memory_ttl: Optional[float]
default_in_redis_ttl: Optional[float]
- max_connections: Optional[Int]
# Type of cache (options: "local", "redis", "s3")
type: s3
diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md
index f70701886b5..974e95a07bd 100644
--- a/docs/my-website/docs/proxy/config_settings.md
+++ b/docs/my-website/docs/proxy/config_settings.md
@@ -50,7 +50,6 @@ litellm_settings:
port: 6379 # The port number for the Redis cache. Required if type is "redis".
password: "your_password" # The password for the Redis cache. Required if type is "redis".
namespace: "litellm.caching.caching" # namespace for redis cache
- max_connections: 100 # [OPTIONAL] Set Maximum number of Redis connections. Passed directly to redis-py.
# Optional - Redis Cluster Settings
redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}]
diff --git a/docs/my-website/docs/proxy/custom_auth.md b/docs/my-website/docs/proxy/custom_auth.md
index 3787f9bdd7c..812b80d3e9c 100644
--- a/docs/my-website/docs/proxy/custom_auth.md
+++ b/docs/my-website/docs/proxy/custom_auth.md
@@ -21,6 +21,169 @@ async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
raise Exception
```
+## UserAPIKeyAuth Fields Reference
+
+The `UserAPIKeyAuth` object supports the following fields for comprehensive auth configuration:
+
+### Core Authentication Fields
+```python
+UserAPIKeyAuth(
+ # Basic auth fields
+ api_key: Optional[str] = None, # The API key (will be hashed automatically)
+ token: Optional[str] = None, # Hashed token for internal use
+ key_name: Optional[str] = None, # Human-readable key name
+ key_alias: Optional[str] = None, # Key alias for identification
+
+ # User identification
+ user_id: Optional[str] = None, # Unique user identifier
+ user_email: Optional[str] = None, # User email address
+ user_role: Optional[LitellmUserRoles] = None, # User role (PROXY_ADMIN, INTERNAL_USER, etc.)
+
+ # Team/Organization
+ team_id: Optional[str] = None, # Team identifier
+ team_alias: Optional[str] = None, # Team display name
+ org_id: Optional[str] = None, # Organization identifier
+)
+```
+
+### Budget and Spend Tracking
+```python
+UserAPIKeyAuth(
+ # User budgets
+ max_budget: Optional[float] = None, # Maximum budget for the key
+ spend: float = 0.0, # Current spend amount
+ soft_budget: Optional[float] = None, # Soft budget limit (warnings)
+ model_max_budget: Dict = {}, # Per-model budget limits
+ model_spend: Dict = {}, # Per-model spend tracking
+
+ # Team budgets
+ team_max_budget: Optional[float] = None, # Team's maximum budget
+ team_spend: Optional[float] = None, # Team's current spend
+ team_member_spend: Optional[float] = None, # This user's spend within the team
+
+ # Budget timing
+ budget_duration: Optional[str] = None, # Budget reset period
+ budget_reset_at: Optional[datetime] = None, # When budget resets
+)
+```
+
+### Rate Limiting
+```python
+UserAPIKeyAuth(
+ # User limits
+ tpm_limit: Optional[int] = None, # Tokens per minute limit
+ rpm_limit: Optional[int] = None, # Requests per minute limit
+ user_tpm_limit: Optional[int] = None, # User-specific TPM limit
+ user_rpm_limit: Optional[int] = None, # User-specific RPM limit
+
+ # Team limits
+ team_tpm_limit: Optional[int] = None, # Team TPM limit
+ team_rpm_limit: Optional[int] = None, # Team RPM limit
+ team_member_tpm_limit: Optional[int] = None, # Per-member TPM limit
+ team_member_rpm_limit: Optional[int] = None, # Per-member RPM limit
+
+ # Per-model limits
+ rpm_limit_per_model: Optional[Dict[str, int]] = None, # RPM limits by model
+ tpm_limit_per_model: Optional[Dict[str, int]] = None, # TPM limits by model
+)
+```
+
+### End User Tracking
+```python
+UserAPIKeyAuth(
+ # End user identification and limits
+ end_user_id: Optional[str] = None, # End user identifier
+ end_user_tpm_limit: Optional[int] = None, # End user TPM limit
+ end_user_rpm_limit: Optional[int] = None, # End user RPM limit
+ end_user_max_budget: Optional[float] = None, # End user budget limit
+)
+```
+
+### Model and Route Access
+```python
+UserAPIKeyAuth(
+ # Model access control
+ models: List = [], # Allowed models list
+ team_models: List = [], # Team's allowed models
+ aliases: Dict = {}, # Model aliases
+
+ # Route permissions
+ allowed_routes: Optional[list] = [], # Allowed API routes
+ allowed_cache_controls: Optional[list] = [], # Cache control permissions
+ permissions: Dict = {}, # General permissions
+)
+```
+
+### Advanced Configuration
+```python
+UserAPIKeyAuth(
+ # Request handling
+ max_parallel_requests: Optional[int] = None, # Concurrent request limit
+ allowed_model_region: Optional[AllowedModelRegion] = None, # Geographic restrictions
+
+ # Expiration and status
+ expires: Optional[Union[str, datetime]] = None, # Key expiration
+ blocked: Optional[bool] = None, # Whether key is blocked
+
+ # Metadata and configuration
+ metadata: Dict = {}, # Custom metadata
+ config: Dict = {}, # Configuration settings
+ team_metadata: Optional[Dict] = None, # Team metadata
+
+ # Internal tracking
+ request_route: Optional[str] = None, # Current request route
+ last_refreshed_at: Optional[float] = None, # Cache refresh timestamp
+)
+```
+
+### Complete Example
+
+```python
+from datetime import datetime, timedelta
+from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles
+
+async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
+ try:
+ # Example: Comprehensive auth configuration
+ if api_key.startswith("sk-admin-"):
+ return UserAPIKeyAuth(
+ api_key=api_key,
+ user_id="admin_user_123",
+ user_email="admin@company.com",
+ user_role=LitellmUserRoles.PROXY_ADMIN,
+ team_id="admin_team",
+ team_alias="Administrative Team",
+ max_budget=1000.0,
+ soft_budget=800.0,
+ tpm_limit=10000,
+ rpm_limit=100,
+ models=["gpt-4", "claude-3-sonnet", "gpt-3.5-turbo"],
+ allowed_routes=["/chat/completions", "/embeddings"],
+ expires=datetime.now() + timedelta(days=30),
+ metadata={"department": "engineering", "cost_center": "ai_ops"}
+ )
+ elif api_key.startswith("sk-team-"):
+ return UserAPIKeyAuth(
+ api_key=api_key,
+ user_id="team_user_456",
+ user_email="user@company.com",
+ user_role=LitellmUserRoles.INTERNAL_USER,
+ team_id="dev_team",
+ team_alias="Development Team",
+ max_budget=100.0,
+ tpm_limit=1000,
+ rpm_limit=20,
+ models=["gpt-3.5-turbo", "claude-3-haiku"],
+ team_member_tpm_limit=500, # Limit within team
+ end_user_tpm_limit=100, # Per end-user limit
+ metadata={"project": "chatbot_v2"}
+ )
+ else:
+ raise Exception("Invalid API key")
+ except Exception:
+ raise Exception("Authentication failed")
+```
+
#### 2. Pass the filepath (relative to the config.yaml)
Pass the filepath to the config.yaml
diff --git a/docs/my-website/docs/proxy/custom_sso.md b/docs/my-website/docs/proxy/custom_sso.md
index bbd7f41bee1..8e869a11393 100644
--- a/docs/my-website/docs/proxy/custom_sso.md
+++ b/docs/my-website/docs/proxy/custom_sso.md
@@ -1,7 +1,9 @@
# ✨ Event Hooks for SSO Login
:::info
-✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
+
+✨ This is an Enterprise only feature [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
+
:::
## Overview
diff --git a/docs/my-website/docs/proxy/db_deadlocks.md b/docs/my-website/docs/proxy/db_deadlocks.md
index ef9d31d6232..0eee928fa64 100644
--- a/docs/my-website/docs/proxy/db_deadlocks.md
+++ b/docs/my-website/docs/proxy/db_deadlocks.md
@@ -84,29 +84,3 @@ LiteLLM emits the following prometheus metrics to monitor the health/status of t
| `litellm_in_memory_spend_update_queue_size` | In-memory aggregate spend values for keys, users, teams, team members, etc.| In-Memory |
| `litellm_redis_spend_update_queue_size` | Redis aggregate spend values for keys, users, teams, etc. | Redis |
-
-## Troubleshooting: Redis Connection Errors
-
-You may see errors like:
-
-```
-LiteLLM Redis Caching: async async_increment() - Got exception from REDIS No connection available., Writing value=21
-LiteLLM Redis Caching: async set_cache_pipeline() - Got exception from REDIS No connection available., Writing value=None
-```
-
-This means all available Redis connections are in use, and LiteLLM cannot obtain a new connection from the pool. This can happen under high load or with many concurrent proxy requests.
-
-**Solution:**
-
-- Increase the `max_connections` parameter in your Redis config section in `proxy_config.yaml` to allow more simultaneous connections. For example:
-
-```yaml
-litellm_settings:
- cache: True
- cache_params:
- type: redis
- max_connections: 100 # Increase as needed for your traffic
-```
-
-Adjust this value based on your expected concurrency and Redis server capacity.
-
diff --git a/docs/my-website/docs/proxy/dynamic_rate_limit.md b/docs/my-website/docs/proxy/dynamic_rate_limit.md
index 74a2d4c6c1b..b3aed6a359e 100644
--- a/docs/my-website/docs/proxy/dynamic_rate_limit.md
+++ b/docs/my-website/docs/proxy/dynamic_rate_limit.md
@@ -139,6 +139,8 @@ litellm_settings:
priority_reservation:
"prod": 0.9 # 90% reserved for production (9 RPM)
"dev": 0.1 # 10% reserved for development (1 RPM)
+ priority_reservation_settings:
+ default_priority: 0 # Weight (0%) assigned to keys without explicit priority metadata
general_settings:
master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
@@ -152,6 +154,9 @@ general_settings:
- **Value (float)**: Percentage of total TPM/RPM to reserve (0.0 to 1.0)
- **Note**: Values should sum to 1.0 or less
+`priority_reservation_settings`: Object (Optional)
+- **default_priority (float)**: Weight/percentage (0.0 to 1.0) assigned to API keys that have no priority metadata set (defaults to 0.5)
+
**Start Proxy**
```bash
@@ -180,6 +185,14 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \
}'
```
+**Key Without Priority (uses default_priority weight):**
+```bash
+curl -X POST 'http://0.0.0.0:4000/key/generate' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{}'
+```
+
**Expected Response for both:**
```json
{
@@ -217,9 +230,10 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
With the configuration above:
-1. **Production keys** can make up to 9 requests per minute
-2. **Development keys** can make up to 1 request per minute
-3. Production requests are never blocked by development usage
+1. **Production keys** can make up to 9 requests per minute (90% of 10 RPM)
+2. **Development keys** can make up to 1 request per minute (10% of 10 RPM)
+3. **Keys without explicit priority** get the default_priority weight (0 = 0%), which allocates 0 requests per minute (0% of 10 RPM)
+4. Named priorities in `priority_reservation` and keys with `default_priority` operate independently
**Rate Limit Error Example:**
```json
diff --git a/docs/my-website/docs/proxy/guardrails/bedrock.md b/docs/my-website/docs/proxy/guardrails/bedrock.md
index 4a1a0a246f8..6725acf1f25 100644
--- a/docs/my-website/docs/proxy/guardrails/bedrock.md
+++ b/docs/my-website/docs/proxy/guardrails/bedrock.md
@@ -4,10 +4,6 @@ import TabItem from '@theme/TabItem';
# Bedrock Guardrails
-:::tip ⚡️
-If you haven't set up or authenticated your Bedrock provider yet, see the [Bedrock Provider Setup & Authentication Guide](../../providers/bedrock.md).
-:::
-
LiteLLM supports Bedrock guardrails via the [Bedrock ApplyGuardrail API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ApplyGuardrail.html).
## Quick Start
diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md
index 54c917bbbca..bcbc4e93651 100644
--- a/docs/my-website/docs/proxy/load_balancing.md
+++ b/docs/my-website/docs/proxy/load_balancing.md
@@ -172,9 +172,6 @@ router_settings:
redis_host:
redis_password:
redis_port: 1992
- cache_params:
- type: redis
- max_connections: 100 # maximum Redis connections in the pool; tune based on expected concurrency/load
```
## Router settings on config - routing_strategy, model_group_alias
diff --git a/docs/my-website/docs/proxy/self_serve.md b/docs/my-website/docs/proxy/self_serve.md
index b54344c1d05..dff55a8ac04 100644
--- a/docs/my-website/docs/proxy/self_serve.md
+++ b/docs/my-website/docs/proxy/self_serve.md
@@ -227,7 +227,7 @@ export PROXY_LOGOUT_URL="https://www.google.com"
-### Set default max budget for internal users
+### Set max budget for internal users
Automatically apply budget per internal user when they sign up. By default the table will be checked every 10 minutes, for users to reset. To modify this, [see this](./users.md#reset-budgets)
@@ -239,10 +239,6 @@ litellm_settings:
This sets a max budget of $10 USD for internal users when they sign up.
-You can also manage these settings visually in the UI:
-
-
-
This budget only applies to personal keys created by that user - seen under `Default Team` on the UI.
diff --git a/docs/my-website/docs/proxy/user_onboarding.md b/docs/my-website/docs/proxy/user_onboarding.md
new file mode 100644
index 00000000000..baa241d6cdf
--- /dev/null
+++ b/docs/my-website/docs/proxy/user_onboarding.md
@@ -0,0 +1,82 @@
+# User Onboarding Guide
+
+A step-by-step guide to help admins onboard users to your LiteLLM proxy instance and help users get started with their API key.
+
+---
+
+## For Administrators
+
+### Step 1: Create a User Account
+
+You can create a user account via the Admin UI or using the API.
+
+#### Admin UI
+- Go to the (`/ui` endpoint)
+- Navigate to the Internal Users section
+- Click "Add User" and fill in the required details
+
+#### API
+```bash
+curl -X POST http://localhost:4000/user/new \
+ -H "Authorization: Bearer " \
+ -H "Content-Type: application/json" \
+ -d '{"user_email": "user@example.com"}'
+```
+
+---
+
+### Step 2: Grant Access & Permissions
+
+- Assign the user to a team (optional)
+- Set budgets, rate limits, and allowed models as needed
+- Generate an API key for the user (via UI or API)
+
+#### **Generate API Key (API Example)**
+```bash
+curl -X POST http://localhost:4000/key/generate \
+ -H "Authorization: Bearer " \
+ -H "Content-Type: application/json" \
+ -d '{"user_id": "", "max_budget": 100}'
+```
+
+---
+
+## For End Users
+
+### Step 3: Validate Your API Key
+
+Before making LLM calls, validate your key works by calling the `/v1/models` endpoint:
+
+```bash
+curl -X GET http://localhost:4000/v1/models \
+ -H "Authorization: Bearer "
+```
+- If your key is valid, you'll get a list of available models.
+- If invalid, you'll get a 401 error.
+
+---
+
+### Step 4: Hello World - Make Your First LLM Call
+
+```bash
+curl -X POST http://localhost:4000/v1/chat/completions \
+ -H "Authorization: Bearer " \
+ -H "Content-Type: application/json" \
+ -d '{
+ "model": "gpt-3.5-turbo",
+ "messages": [{"role": "user", "content": "Hello!"}]
+ }'
+```
+
+---
+
+## Troubleshooting
+- If you get a 401 error, check with your admin that your key is active and you have access to the requested model.
+- Use the `/v1/models` endpoint to quickly check if your key is valid without consuming LLM tokens.
+
+---
+
+## See Also
+- [Proxy Quick Start](./quick_start.md)
+- [User Management](./users.md)
+- [Key Management](./key_management.md)
diff --git a/docs/my-website/docs/proxy_api.md b/docs/my-website/docs/proxy_api.md
index 7612645fb54..89bfacbe19f 100644
--- a/docs/my-website/docs/proxy_api.md
+++ b/docs/my-website/docs/proxy_api.md
@@ -27,7 +27,7 @@ Email us @ krrish@berri.ai
## Supported Models for LiteLLM Key
These are the models that currently work with the "sk-litellm-.." keys.
-For a complete list of models/providers that you can call with LiteLLM, [check out our provider list](./providers/) or check out [models.litellm.ai](https://models.litellm.ai/)
+For a complete list of models/providers that you can call with LiteLLM, [check out our provider list](./providers/)
* OpenAI models - [OpenAI docs](./providers/openai.md)
* gpt-4
diff --git a/docs/my-website/docs/rerank.md b/docs/my-website/docs/rerank.md
index cad64718384..c57eacbb224 100644
--- a/docs/my-website/docs/rerank.md
+++ b/docs/my-website/docs/rerank.md
@@ -109,8 +109,6 @@ curl http://0.0.0.0:4000/rerank \
## **Supported Providers**
-#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
-
| Provider | Link to Usage |
|-------------|--------------------|
| Cohere (v1 + v2 clients) | [Usage](#quick-start) |
diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md
index b03e4f8be92..94d7c73be05 100644
--- a/docs/my-website/docs/response_api.md
+++ b/docs/my-website/docs/response_api.md
@@ -3,11 +3,8 @@ import TabItem from '@theme/TabItem';
# /responses [Beta]
-
LiteLLM provides a BETA endpoint in the spec of [OpenAI's `/responses` API](https://platform.openai.com/docs/api-reference/responses)
-Requests to /chat/completions may be bridged here automatically when the provider lacks support for that endpoint. The model’s default `mode` determines how bridging works.(see `model_prices_and_context_window`)
-
| Feature | Supported | Notes |
|---------|-----------|--------|
| Cost Tracking | ✅ | Works with all supported models |
@@ -81,43 +78,6 @@ print(retrieved_response)
# retrieved_response = await litellm.aget_responses(response_id=response_id)
```
-#### CANCEL a Response
-You can cancel an in-progress response (if supported by the provider):
-
-```python showLineNumbers title="Cancel Response by ID"
-import litellm
-
-# First, create a response
-response = litellm.responses(
- model="openai/o1-pro",
- input="Tell me a three sentence bedtime story about a unicorn.",
- max_output_tokens=100
-)
-
-# Get the response ID
-response_id = response.id
-
-# Cancel the response by ID
-cancel_response = litellm.cancel_responses(
- response_id=response_id
-)
-
-print(cancel_response)
-
-# For async usage
-# cancel_response = await litellm.acancel_responses(response_id=response_id)
-```
-
-
-**REST API:**
-```bash
-curl -X POST http://localhost:4000/v1/responses/response_id/cancel \
- -H "Authorization: Bearer sk-1234"
-```
-
-This will attempt to cancel the in-progress response with the given ID.
-**Note:** Not all providers support response cancellation. If unsupported, an error will be raised.
-
#### DELETE a Response
```python showLineNumbers title="Delete Response by ID"
import litellm
diff --git a/docs/my-website/docs/sdk/headers.md b/docs/my-website/docs/sdk/headers.md
deleted file mode 100644
index a1fdaadc6f5..00000000000
--- a/docs/my-website/docs/sdk/headers.md
+++ /dev/null
@@ -1,328 +0,0 @@
-# SDK Header Support
-
-LiteLLM SDK provides comprehensive support for passing additional headers with API requests. This is essential for enterprise environments using API gateways, service meshes, and multi-tenant architectures.
-
-## Overview
-
-Headers can be passed to LiteLLM in three ways, with the following priority order:
-1. **Request-specific headers** (highest priority)
-2. **extra_headers parameter**
-3. **Global litellm.headers** (lowest priority)
-
-When the same header key is specified in multiple places, the higher priority value will be used.
-
-## Usage Methods
-
-### 1. Global Headers (litellm.headers)
-
-Set headers that will be included in all API requests:
-
-```python
-import litellm
-
-# Set global headers for all requests
-litellm.headers = {
- "X-API-Gateway-Key": "your-gateway-key",
- "X-Company-ID": "acme-corp",
- "X-Environment": "production"
-}
-
-# Now all completion calls will include these headers
-response = litellm.completion(
- model="claude-3-5-sonnet-latest",
- messages=[{"role": "user", "content": "Hello"}]
-)
-```
-
-### 2. Per-Request Headers (extra_headers)
-
-Pass headers for specific requests using the `extra_headers` parameter:
-
-```python
-import litellm
-
-response = litellm.completion(
- model="gpt-4",
- messages=[{"role": "user", "content": "Hello"}],
- extra_headers={
- "X-Request-ID": "req-12345",
- "X-Tenant-ID": "tenant-abc",
- "X-Custom-Auth": "bearer-token-xyz"
- }
-)
-```
-
-### 3. Request Headers (headers parameter)
-
-Use the `headers` parameter for the highest priority header control:
-
-```python
-import litellm
-
-response = litellm.completion(
- model="claude-3-5-sonnet-latest",
- messages=[{"role": "user", "content": "Hello"}],
- headers={
- "X-Priority-Header": "high-priority-value",
- "Authorization": "Bearer custom-token"
- }
-)
-```
-
-### 4. Combining All Methods
-
-You can combine all three methods. Headers will be merged with the priority order:
-
-```python
-import litellm
-
-# Global headers (lowest priority)
-litellm.headers = {
- "X-Company-ID": "acme-corp",
- "X-Shared-Header": "global-value"
-}
-
-response = litellm.completion(
- model="gpt-4",
- messages=[{"role": "user", "content": "Hello"}],
- extra_headers={
- "X-Request-ID": "req-12345",
- "X-Shared-Header": "extra-value" # Overrides global
- },
- headers={
- "X-Priority-Header": "important",
- "X-Shared-Header": "request-value" # Overrides both global and extra
- }
-)
-
-# Final headers sent to API:
-# {
-# "X-Company-ID": "acme-corp", # From global
-# "X-Request-ID": "req-12345", # From extra_headers
-# "X-Priority-Header": "important", # From headers
-# "X-Shared-Header": "request-value" # From headers (highest priority)
-# }
-```
-
-## Enterprise Use Cases
-
-### API Gateway Integration (Apigee, Kong, AWS API Gateway)
-
-```python
-import litellm
-
-# Set up headers for API gateway routing and authentication
-litellm.headers = {
- "X-API-Gateway-Key": "your-gateway-key",
- "X-Route-Version": "v2"
-}
-
-# Per-tenant requests
-response = litellm.completion(
- model="claude-3-5-sonnet-latest",
- messages=[{"role": "user", "content": "Analyze this data"}],
- extra_headers={
- "X-Tenant-ID": "tenant-123",
- "X-Department": "engineering"
- }
-)
-```
-
-### Service Mesh (Istio, Linkerd)
-
-```python
-import litellm
-
-response = litellm.completion(
- model="gpt-4",
- messages=[{"role": "user", "content": "Hello"}],
- extra_headers={
- "X-Trace-ID": "trace-abc-123",
- "X-Service-Name": "ai-service",
- "X-Version": "1.2.3"
- }
-)
-```
-
-### Multi-Tenant SaaS Applications
-
-```python
-import litellm
-
-def make_ai_request(user_id, tenant_id, content):
- return litellm.completion(
- model="claude-3-5-sonnet-latest",
- messages=[{"role": "user", "content": content}],
- extra_headers={
- "X-User-ID": user_id,
- "X-Tenant-ID": tenant_id,
- "X-Request-Time": str(int(time.time()))
- }
- )
-
-# Usage
-response = make_ai_request("user-456", "tenant-org-1", "Help me write code")
-```
-
-### Request Tracing and Debugging
-
-```python
-import litellm
-import uuid
-
-def traced_completion(model, messages, **kwargs):
- trace_id = str(uuid.uuid4())
-
- return litellm.completion(
- model=model,
- messages=messages,
- extra_headers={
- "X-Trace-ID": trace_id,
- "X-Debug-Mode": "true",
- "X-Source-Service": "my-app"
- },
- **kwargs
- )
-
-# Usage
-response = traced_completion(
- model="gpt-4",
- messages=[{"role": "user", "content": "Debug this issue"}]
-)
-```
-
-### Custom Authentication
-
-```python
-import litellm
-
-def get_custom_auth_token():
- # Your custom authentication logic
- return "custom-auth-token"
-
-response = litellm.completion(
- model="claude-3-5-sonnet-latest",
- messages=[{"role": "user", "content": "Hello"}],
- headers={
- "X-Custom-Auth": get_custom_auth_token(),
- "X-Auth-Type": "custom"
- }
-)
-```
-
-## Provider Support
-
-Headers are supported across all LiteLLM providers including:
-
-- **OpenAI** (GPT models)
-- **Anthropic** (Claude models)
-- **Cohere**
-- **Hugging Face**
-- **Custom providers**
-- **Azure OpenAI**
-- **AWS Bedrock**
-- **Google Vertex AI**
-
-Each provider will receive your custom headers along with their required authentication and API-specific headers.
-
-## Best Practices
-
-### 1. Use Meaningful Header Names
-```python
-# Good
-extra_headers = {
- "X-Request-ID": "req-12345",
- "X-Tenant-ID": "org-456"
-}
-
-# Avoid
-extra_headers = {
- "custom1": "value1",
- "h2": "value2"
-}
-```
-
-### 2. Include Tracing Information
-```python
-extra_headers = {
- "X-Trace-ID": trace_id,
- "X-Span-ID": span_id,
- "X-Service-Name": "ai-service"
-}
-```
-
-### 3. Handle Sensitive Information Carefully
-```python
-# Don't log sensitive headers
-import os
-
-if os.getenv("ENVIRONMENT") != "production":
- extra_headers["X-Debug-User"] = user_id
-```
-
-### 4. Use Environment-Specific Headers
-```python
-import os
-
-environment = os.getenv("ENVIRONMENT", "development")
-
-litellm.headers = {
- "X-Environment": environment,
- "X-Service-Version": os.getenv("SERVICE_VERSION", "unknown")
-}
-```
-
-## Troubleshooting
-
-### Headers Not Being Passed
-
-If your headers aren't reaching the API:
-
-1. **Check Header Names**: Ensure header names don't conflict with provider-specific headers
-2. **Verify Priority**: Remember that `headers` > `extra_headers` > `litellm.headers`
-3. **Test with Logging**: Enable verbose logging to see what headers are being sent
-
-```python
-import litellm
-
-# Enable debug logging
-litellm.set_verbose = True
-
-response = litellm.completion(
- model="gpt-4",
- messages=[{"role": "user", "content": "test"}],
- extra_headers={"X-Debug": "test"}
-)
-```
-
-### Gateway or Proxy Issues
-
-If using API gateways or proxies:
-
-1. **Check Gateway Requirements**: Verify required headers for your gateway
-2. **Test Direct vs Gateway**: Compare direct API calls vs gateway calls
-3. **Validate Header Format**: Some gateways have header format requirements
-
-## Security Considerations
-
-1. **Don't Log Sensitive Headers**: Avoid logging authentication tokens or personal data
-2. **Use HTTPS**: Always use secure connections when passing sensitive headers
-3. **Validate Header Values**: Sanitize user-provided header values
-4. **Rotate Keys**: Regularly rotate any API keys passed in headers
-
-```python
-import litellm
-import re
-
-def safe_header_value(value):
- # Remove potentially dangerous characters
- return re.sub(r'[^\w\-.]', '', str(value))
-
-response = litellm.completion(
- model="gpt-4",
- messages=[{"role": "user", "content": "Hello"}],
- extra_headers={
- "X-User-ID": safe_header_value(user_id)
- }
-)
-```
\ No newline at end of file
diff --git a/docs/my-website/img/default_user_settings_admin_ui.png b/docs/my-website/img/default_user_settings_admin_ui.png
deleted file mode 100644
index 5910154cd51..00000000000
Binary files a/docs/my-website/img/default_user_settings_admin_ui.png and /dev/null differ
diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js
index fc14c47a157..a131e5c34ec 100644
--- a/docs/my-website/sidebars.js
+++ b/docs/my-website/sidebars.js
@@ -57,31 +57,32 @@ const sidebars = {
type: "category",
label: "Alerting & Monitoring",
items: [
+ "proxy/prometheus",
"proxy/alerting",
- "proxy/pagerduty",
- "proxy/prometheus"
- ]
+ "proxy/pagerduty"
+ ].sort()
},
{
type: "category",
label: "[Beta] Prompt Management",
items: [
- "proxy/custom_prompt_management",
+ "proxy/prompt_management",
"proxy/native_litellm_prompt",
- "proxy/prompt_management"
- ]
+ "proxy/custom_prompt_management"
+ ].sort()
},
{
type: "category",
label: "AI Tools (OpenWebUI, Claude Code, etc.)",
items: [
- "tutorials/claude_responses_api",
- "tutorials/cost_tracking_coding",
- "tutorials/github_copilot_integration",
+ "integrations/letta",
+ "tutorials/openweb_ui",
+ "tutorials/openai_codex",
"tutorials/litellm_gemini_cli",
"tutorials/litellm_qwen_code_cli",
- "tutorials/openai_codex",
- "tutorials/openweb_ui"
+ "tutorials/github_copilot_integration",
+ "tutorials/claude_responses_api",
+ "tutorials/cost_tracking_coding",
]
},
@@ -111,115 +112,29 @@ const sidebars = {
label: "Setup & Deployment",
items: [
"proxy/quick_start",
- "proxy/cli",
- "proxy/debugging",
+ "proxy/user_onboarding",
"proxy/deploy",
- "proxy/health",
- "proxy/master_key_rotations",
- "proxy/model_management",
"proxy/prod",
+ "proxy/cli",
"proxy/release_cycle",
+ "proxy/model_management",
+ "proxy/health",
+ "proxy/debugging",
+ "proxy/master_key_rotations",
],
},
"proxy/demo",
- {
- type: "category",
- label: "Admin UI",
- items: [
- "proxy/admin_ui_sso",
- "proxy/custom_root_ui",
- "proxy/custom_sso",
- "proxy/model_hub",
- "proxy/public_teams",
- "proxy/self_serve",
- "proxy/ui",
- "proxy/ui/bulk_edit_users",
- "proxy/ui_credentials",
- "tutorials/scim_litellm",
- {
- type: "category",
- label: "UI Logs",
- items: [
- "proxy/ui_logs",
- "proxy/ui_logs_sessions"
- ]
- }
- ],
- },
{
type: "category",
label: "Architecture",
- items: [
- "proxy/architecture",
- "proxy/control_plane_and_data_plane",
- "proxy/db_deadlocks",
- "proxy/db_info",
- "proxy/image_handling",
- "proxy/jwt_auth_arch",
- "proxy/spend_logs_deletion",
- "proxy/user_management_heirarchy",
- "router_architecture"
- ],
+ items: ["proxy/architecture", "proxy/control_plane_and_data_plane", "proxy/db_info", "proxy/db_deadlocks", "router_architecture", "proxy/user_management_heirarchy", "proxy/jwt_auth_arch", "proxy/image_handling", "proxy/spend_logs_deletion"],
},
{
type: "link",
label: "All Endpoints (Swagger)",
href: "https://litellm-api.up.railway.app/",
},
- "proxy/enterprise",
- "proxy/management_cli",
- {
- type: "category",
- label: "Authentication",
- items: [
- "proxy/virtual_keys",
- "proxy/token_auth",
- "proxy/service_accounts",
- "proxy/access_control",
- "proxy/cli_sso",
- "proxy/custom_auth",
- "proxy/ip_address",
- "proxy/email",
- "proxy/multiple_admins",
- ],
- },
- {
- type: "category",
- label: "Budgets + Rate Limits",
- items: [
- "proxy/customers",
- "proxy/dynamic_rate_limit",
- "proxy/rate_limit_tiers",
- "proxy/team_budgets",
- "proxy/temporary_budget_increase",
- "proxy/users"
- ],
- },
- "proxy/caching",
- {
- type: "category",
- label: "Create Custom Plugins",
- description: "Modify requests, responses, and more",
- items: [
- "proxy/call_hooks",
- "proxy/rules",
- ]
- },
- {
- type: "link",
- label: "Load Balancing, Routing, Fallbacks",
- href: "https://docs.litellm.ai/docs/routing-load-balancing",
- },
- {
- type: "category",
- label: "Logging, Alerting, Metrics",
- items: [
- "proxy/dynamic_logging",
- "proxy/logging",
- "proxy/logging_spec",
- "proxy/team_logging"
- ],
- },
+ "proxy/management_cli",
{
type: "category",
label: "Making LLM Requests",
@@ -232,6 +147,19 @@ const sidebars = {
"proxy/model_discovery",
],
},
+ {
+ type: "category",
+ label: "Authentication",
+ items: [
+ "proxy/virtual_keys",
+ "proxy/token_auth",
+ "proxy/service_accounts",
+ "proxy/access_control",
+ "proxy/ip_address",
+ "proxy/email",
+ "proxy/custom_auth",
+ ],
+ },
{
type: "category",
label: "Model Access",
@@ -240,6 +168,73 @@ const sidebars = {
"proxy/team_model_add"
]
},
+ {
+ type: "category",
+ label: "Spend Tracking",
+ items: ["proxy/cost_tracking", "proxy/custom_pricing", "proxy/billing",],
+ },
+ {
+ type: "category",
+ label: "Budgets + Rate Limits",
+ items: ["proxy/users", "proxy/temporary_budget_increase", "proxy/rate_limit_tiers", "proxy/team_budgets", "proxy/dynamic_rate_limit", "proxy/customers"],
+ },
+ {
+ type: "category",
+ label: "Enterprise Features",
+ items: [
+ "proxy/enterprise",
+ {
+ type: "category",
+ label: "Admin UI",
+ items: [
+ "proxy/ui",
+ "proxy/admin_ui_sso",
+ "proxy/custom_root_ui",
+ "proxy/model_hub",
+ "proxy/self_serve",
+ "proxy/public_teams",
+ "proxy/ui_credentials",
+ "proxy/ui/bulk_edit_users",
+ {
+ type: "category",
+ label: "UI Logs",
+ items: [
+ "proxy/ui_logs",
+ "proxy/ui_logs_sessions"
+ ]
+ }
+ ],
+ },
+ {
+ type: "category",
+ label: "SSO & Identity Management",
+ items: [
+ "proxy/cli_sso",
+ "proxy/admin_ui_sso",
+ "proxy/custom_sso",
+ "tutorials/scim_litellm",
+ "tutorials/msft_sso",
+ "proxy/multiple_admins",
+ ],
+ },
+ ],
+ },
+ {
+ type: "link",
+ label: "Load Balancing, Routing, Fallbacks",
+ href: "https://docs.litellm.ai/docs/routing-load-balancing",
+ },
+ {
+ type: "category",
+ label: "Logging, Alerting, Metrics",
+ items: [
+ "proxy/logging",
+ "proxy/logging_spec",
+ "proxy/team_logging",
+ "proxy/dynamic_logging"
+ ],
+ },
+
{
type: "category",
label: "Secret Managers",
@@ -250,13 +245,14 @@ const sidebars = {
},
{
type: "category",
- label: "Spend Tracking",
+ label: "Create Custom Plugins",
+ description: "Modify requests, responses, and more",
items: [
- "proxy/billing",
- "proxy/cost_tracking",
- "proxy/custom_pricing"
- ],
+ "proxy/call_hooks",
+ "proxy/rules",
+ ]
},
+ "proxy/caching",
]
},
{
@@ -270,11 +266,13 @@ const sidebars = {
slug: "/supported_endpoints",
},
items: [
+ "anthropic_unified",
+ "apply_guardrail",
"assistants",
{
type: "category",
label: "/audio",
- items: [
+ "items": [
"audio_transcription",
"text_to_speech",
]
@@ -303,7 +301,6 @@ const sidebars = {
"completion/http_handler_config",
],
},
- "text_completion",
"embedding/supported_embedding",
{
type: "category",
@@ -321,14 +318,13 @@ const sidebars = {
"proxy/managed_finetuning",
]
},
- "generateContent",
- "apply_guardrail",
+ "generateContent",
{
type: "category",
label: "/images",
items: [
- "image_edits",
"image_generation",
+ "image_edits",
"image_variations",
]
},
@@ -339,23 +335,23 @@ const sidebars = {
label: "Pass-through Endpoints (Anthropic SDK, etc.)",
items: [
"pass_through/intro",
- "pass_through/anthropic_completion",
- "pass_through/assembly_ai",
- "pass_through/bedrock",
- "pass_through/cohere",
+ "pass_through/vertex_ai",
"pass_through/google_ai_studio",
- "pass_through/langfuse",
+ "pass_through/cohere",
+ "pass_through/vllm",
"pass_through/mistral",
"pass_through/openai_passthrough",
- "pass_through/vertex_ai",
- "pass_through/vllm",
- "proxy/pass_through"
- ]
+ "pass_through/anthropic_completion",
+ "pass_through/bedrock",
+ "pass_through/assembly_ai",
+ "pass_through/langfuse",
+ "proxy/pass_through",
+ ],
},
"realtime",
"rerank",
"response_api",
- "anthropic_unified",
+ "text_completion",
{
type: "category",
label: "/vector_stores",
@@ -519,32 +515,33 @@ const sidebars = {
type: "category",
label: "Guides",
items: [
- "completion/audio",
- "completion/batching",
- "completion/computer_use",
- "completion/document_understanding",
- "completion/drop_params",
- "completion/function_call",
- "completion/image_generation_chat",
- "completion/json_mode",
- "completion/knowledgebase",
- "completion/message_trimming",
- "completion/model_alias",
- "completion/mock_requests",
- "completion/predict_outputs",
- "completion/prefix",
- "completion/prompt_caching",
- "completion/prompt_formatting",
- "completion/reliable_completions",
- "completion/stream",
- "completion/provider_specific_params",
- "completion/vision",
- "completion/web_search",
"exception_mapping",
+ "completion/provider_specific_params",
"guides/finetuned_models",
"guides/security_settings",
+ "completion/audio",
+ "completion/image_generation_chat",
+ "completion/web_search",
+ "completion/document_understanding",
+ "completion/vision",
+ "completion/json_mode",
+ "reasoning_content",
+ "completion/computer_use",
+ "completion/prompt_caching",
+ "completion/predict_outputs",
+ "completion/knowledgebase",
+ "completion/prefix",
+ "completion/drop_params",
+ "completion/prompt_formatting",
+ "completion/stream",
+ "completion/message_trimming",
+ "completion/function_call",
+ "completion/model_alias",
+ "completion/batching",
+ "completion/mock_requests",
+ "completion/reliable_completions",
"proxy/veo_video_generation",
- "reasoning_content"
+
]
},
@@ -557,37 +554,26 @@ const sidebars = {
description: "Learn how to load balance, route, and set fallbacks for your LLM requests",
slug: "/routing-load-balancing",
},
- items: [
- "routing",
- "scheduler",
- "proxy/auto_routing",
- "proxy/load_balancing",
- "proxy/provider_budget_routing",
- "proxy/reliability",
- "proxy/tag_routing",
- "proxy/timeout",
- "wildcard_routing"
- ],
+ items: ["routing", "scheduler", "proxy/load_balancing", "proxy/reliability", "proxy/timeout", "proxy/auto_routing", "proxy/tag_routing", "proxy/provider_budget_routing", "wildcard_routing"],
},
{
type: "category",
label: "LiteLLM Python SDK",
items: [
"set_keys",
- "budget_manager",
- "caching/all_caches",
"completion/token_usage",
"sdk/headers",
"sdk_custom_pricing",
"embedding/async_embedding",
"embedding/moderation",
+ "budget_manager",
+ "caching/all_caches",
"migration",
- "sdk_custom_pricing",
{
type: "category",
label: "LangChain, LlamaIndex, Instructor Integration",
items: ["langchain/langchain", "tutorials/instructor"],
- }
+ },
],
},
diff --git a/examples/sdk_headers_example.py b/examples/sdk_headers_example.py
deleted file mode 100644
index 4ce31df4a70..00000000000
--- a/examples/sdk_headers_example.py
+++ /dev/null
@@ -1,195 +0,0 @@
-#!/usr/bin/env python3
-"""
-Example demonstrating LiteLLM SDK header support for enterprise environments.
-
-This example shows how to use additional headers with API gateways, service meshes,
-and multi-tenant architectures.
-"""
-
-import litellm
-import os
-from typing import Dict, Any
-
-def example_global_headers():
- """Example: Set global headers for all requests"""
- print("=== Global Headers Example ===")
-
- # Set global headers that will be included in all API requests
- litellm.headers = {
- "X-API-Gateway-Key": "your-gateway-key-here",
- "X-Company-ID": "acme-corp",
- "X-Environment": "production"
- }
-
- print("Global headers set:", litellm.headers)
-
- # These headers will now be included in all completion calls
- # (Note: This example doesn't actually make API calls)
- print("Global headers will be included in all subsequent completion() calls")
-
-
-def example_per_request_headers():
- """Example: Using extra_headers for specific requests"""
- print("\n=== Per-Request Headers Example ===")
-
- headers_to_send = {
- "X-Request-ID": "req-12345",
- "X-Tenant-ID": "tenant-abc",
- "X-Custom-Auth": "bearer-token-xyz"
- }
-
- print("Per-request headers:", headers_to_send)
-
- # Example of how you would use extra_headers in a real call
- # response = litellm.completion(
- # model="claude-3-5-sonnet-latest",
- # messages=[{"role": "user", "content": "Hello"}],
- # extra_headers=headers_to_send
- # )
-
-
-def example_header_priority():
- """Example: Demonstrating header priority and merging"""
- print("\n=== Header Priority Example ===")
-
- # Set global headers
- litellm.headers = {
- "X-Company-ID": "acme-corp",
- "X-Shared-Header": "global-value"
- }
-
- # Headers that would be sent in a request
- extra_headers = {
- "X-Request-ID": "req-12345",
- "X-Shared-Header": "extra-value" # Overrides global
- }
-
- request_headers = {
- "X-Priority-Header": "important",
- "X-Shared-Header": "request-value" # Overrides both global and extra
- }
-
- print("Global headers:", litellm.headers)
- print("Extra headers:", extra_headers)
- print("Request headers:", request_headers)
- print("\nFinal headers would be:")
- print(" X-Company-ID: acme-corp (from global)")
- print(" X-Request-ID: req-12345 (from extra)")
- print(" X-Priority-Header: important (from request)")
- print(" X-Shared-Header: request-value (request wins - highest priority)")
-
-
-def example_enterprise_api_gateway():
- """Example: Enterprise API Gateway scenario"""
- print("\n=== Enterprise API Gateway Example ===")
-
- # Simulate enterprise environment with Apigee or similar
- gateway_config = {
- "X-API-Gateway-Key": os.getenv("API_GATEWAY_KEY", "demo-key"),
- "X-Route-Version": "v2",
- "X-Rate-Limit-Group": "premium"
- }
-
- # Set gateway headers globally
- litellm.headers = gateway_config
- print("Gateway headers configured:", gateway_config)
-
- # Function to make tenant-specific requests
- def make_tenant_request(tenant_id: str, user_id: str, content: str) -> Dict[str, Any]:
- """Make an AI request with tenant-specific headers"""
-
- tenant_headers = {
- "X-Tenant-ID": tenant_id,
- "X-User-ID": user_id,
- "X-Request-Time": "2024-01-01T00:00:00Z",
- "X-Service-Name": "ai-assistant"
- }
-
- print(f"Making request for tenant {tenant_id}, user {user_id}")
- print("Tenant-specific headers:", tenant_headers)
-
- # In a real scenario, this would make the actual API call:
- # return litellm.completion(
- # model="claude-3-5-sonnet-latest",
- # messages=[{"role": "user", "content": content}],
- # extra_headers=tenant_headers
- # )
-
- # For demo purposes, return mock data
- return {"mock": "response", "headers_used": {**gateway_config, **tenant_headers}}
-
- # Example usage
- result = make_tenant_request("tenant-123", "user-456", "Analyze this data")
- print("Response:", result)
-
-
-def example_service_mesh():
- """Example: Service mesh integration (Istio, Linkerd)"""
- print("\n=== Service Mesh Example ===")
-
- service_mesh_headers = {
- "X-Trace-ID": "trace-abc-123",
- "X-Span-ID": "span-def-456",
- "X-Service-Name": "ai-service",
- "X-Version": "1.2.3",
- "X-Cluster": "prod-us-west-2"
- }
-
- print("Service mesh headers:", service_mesh_headers)
-
- # Example of using these headers for distributed tracing
- # response = litellm.completion(
- # model="gpt-4",
- # messages=[{"role": "user", "content": "Hello"}],
- # extra_headers=service_mesh_headers
- # )
-
-
-def example_debugging_and_monitoring():
- """Example: Request debugging and monitoring"""
- print("\n=== Debugging and Monitoring Example ===")
-
- import uuid
- import time
-
- # Generate unique identifiers for request tracking
- trace_id = str(uuid.uuid4())
- request_id = f"req-{int(time.time())}"
-
- debug_headers = {
- "X-Trace-ID": trace_id,
- "X-Request-ID": request_id,
- "X-Debug-Mode": "true",
- "X-Source-Service": "customer-support-bot",
- "X-Request-Priority": "high"
- }
-
- print("Debug headers:", debug_headers)
- print(f"Trace ID: {trace_id}")
- print(f"Request ID: {request_id}")
-
- # These headers help with:
- # 1. Distributed tracing across services
- # 2. Request correlation in logs
- # 3. Debug mode enablement
- # 4. Priority-based routing
-
-
-if __name__ == "__main__":
- print("LiteLLM SDK Header Support Examples")
- print("=" * 50)
-
- example_global_headers()
- example_per_request_headers()
- example_header_priority()
- example_enterprise_api_gateway()
- example_service_mesh()
- example_debugging_and_monitoring()
-
- print("\n" + "=" * 50)
- print("All examples completed!")
- print("\nTo use in your application:")
- print("1. Set litellm.headers for global headers")
- print("2. Use extra_headers parameter for request-specific headers")
- print("3. Use headers parameter for highest priority headers")
- print("4. Headers are merged with priority: headers > extra_headers > litellm.headers")
\ No newline at end of file
diff --git a/git_model_armor.py b/git_model_armor.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/litellm/__init__.py b/litellm/__init__.py
index 7a68aa3a8d6..523ee36b865 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -89,6 +89,7 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
LiteLLM_UpperboundKeyGenerateParams,
)
from litellm.types.utils import StandardKeyGenerationConfig, LlmProviders
+from litellm.types.utils import PriorityReservationSettings
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
import httpx
@@ -373,6 +374,7 @@ public_model_groups: Optional[List[str]] = None
public_model_groups_links: Dict[str, str] = {}
#### REQUEST PRIORITIZATION ######
priority_reservation: Optional[Dict[str, float]] = None
+priority_reservation_settings: "PriorityReservationSettings" = PriorityReservationSettings()
######## Networking Settings ########
diff --git a/litellm/_uuid.py b/litellm/_uuid.py
new file mode 100644
index 00000000000..05b1adbf75b
--- /dev/null
+++ b/litellm/_uuid.py
@@ -0,0 +1,23 @@
+"""
+Internal unified UUID helper.
+
+Tries to use fastuuid (performance) and falls back to stdlib uuid if unavailable.
+"""
+
+FASTUUID_AVAILABLE = False
+
+try:
+ import fastuuid as _uuid # type: ignore
+
+ FASTUUID_AVAILABLE = True
+except Exception: # pragma: no cover - fallback path
+ import uuid as _uuid # type: ignore
+
+
+# Expose a module-like alias so callers can use: uuid.uuid4()
+uuid = _uuid
+
+
+def uuid4():
+ """Return a UUID4 using the selected backend."""
+ return uuid.uuid4()
diff --git a/litellm/constants.py b/litellm/constants.py
index 005eb2bb6d0..6e70ae0671f 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -812,6 +812,11 @@ BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[
]
BEDROCK_CONVERSE_MODELS = [
+ "qwen.qwen3-coder-480b-a35b-v1:0",
+ "qwen.qwen3-235b-a22b-2507-v1:0",
+ "qwen.qwen3-coder-30b-a3b-v1:0",
+ "qwen.qwen3-32b-v1:0",
+ "deepseek.v3-v1:0",
"openai.gpt-oss-20b-1:0",
"openai.gpt-oss-120b-1:0",
"anthropic.claude-opus-4-1-20250805-v1:0",
diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py
index 24577731384..69943a0fe4d 100644
--- a/litellm/integrations/langfuse/langfuse.py
+++ b/litellm/integrations/langfuse/langfuse.py
@@ -671,6 +671,7 @@ class LangFuseLogger:
generation_id = None
usage = None
+ usage_details = None
if response_obj is not None:
if (
hasattr(response_obj, "id")
@@ -687,6 +688,11 @@ class LangFuseLogger:
"completion_tokens": _usage_obj.completion_tokens,
"total_cost": cost if self._supports_costs() else None,
}
+ usage_details = LangfuseUsageDetails(input=_usage_obj.prompt_tokens,
+ output=_usage_obj.completion_tokens,
+ cache_creation_input_tokens=_usage_obj.get('cache_creation_input_tokens', 0),
+ cache_read_input_tokens=_usage_obj.get('cache_read_input_tokens', 0))
+
generation_name = clean_metadata.pop("generation_name", None)
if generation_name is None:
# if `generation_name` is None, use sensible default values
@@ -719,6 +725,7 @@ class LangFuseLogger:
"input": input if not mask_input else "redacted-by-litellm",
"output": output if not mask_output else "redacted-by-litellm",
"usage": usage,
+ "usage_details": usage_details,
"metadata": log_requester_metadata(clean_metadata),
"level": level,
"version": clean_metadata.pop("version", None),
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index 65f665041f9..64986970d00 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -26,7 +26,6 @@ from typing import (
cast,
)
-import fastuuid as uuid
from httpx import Response
from pydantic import BaseModel
@@ -38,6 +37,7 @@ from litellm import (
turn_off_message_logging,
)
from litellm._logging import _is_debugging_on, verbose_logger
+from litellm._uuid import uuid
from litellm.batches.batch_utils import _handle_completed_batch
from litellm.caching.caching import DualCache, InMemoryCache
from litellm.caching.caching_handler import LLMCachingHandler
diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py
index 322691e28b4..bc0fbdf5c11 100644
--- a/litellm/litellm_core_utils/streaming_handler.py
+++ b/litellm/litellm_core_utils/streaming_handler.py
@@ -1619,11 +1619,12 @@ class CustomStreamWrapper:
completion_start_time=datetime.datetime.now()
)
## LOGGING
- executor.submit(
- self.run_success_logging_and_cache_storage,
- response,
- cache_hit,
- ) # log response
+ if not litellm.disable_streaming_logging:
+ executor.submit(
+ self.run_success_logging_and_cache_storage,
+ response,
+ cache_hit,
+ ) # log response
choice = response.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""
diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py
index e54aeaf995f..6d41655674c 100644
--- a/litellm/llms/anthropic/chat/transformation.py
+++ b/litellm/llms/anthropic/chat/transformation.py
@@ -804,7 +804,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if content.get("citations") is not None:
if citations is None:
citations = []
- citations.append(content["citations"])
+ citations.append(
+ [
+ {
+ **citation,
+ "supported_text": content.get("text", ""),
+ }
+ for citation in content["citations"]
+ ]
+ )
if thinking_blocks is not None:
reasoning_content = ""
for block in thinking_blocks:
diff --git a/litellm/llms/azure/chat/o_series_handler.py b/litellm/llms/azure/chat/o_series_handler.py
index 2f3e9e63996..d0f5153b0eb 100644
--- a/litellm/llms/azure/chat/o_series_handler.py
+++ b/litellm/llms/azure/chat/o_series_handler.py
@@ -4,7 +4,7 @@ Handler file for calls to Azure OpenAI's o1/o3 family of models
Written separately to handle faking streaming for o1 and o3 models.
"""
-from typing import Any, Callable, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union
import httpx
@@ -13,6 +13,9 @@ from litellm.types.utils import ModelResponse
from ...openai.openai import OpenAIChatCompletion
from ..common_utils import BaseAzureLLM
+if TYPE_CHECKING:
+ from aiohttp import ClientSession
+
class AzureOpenAIO1ChatCompletion(BaseAzureLLM, OpenAIChatCompletion):
def completion(
@@ -38,6 +41,7 @@ class AzureOpenAIO1ChatCompletion(BaseAzureLLM, OpenAIChatCompletion):
organization: Optional[str] = None,
custom_llm_provider: Optional[str] = None,
drop_params: Optional[bool] = None,
+ shared_session: Optional["ClientSession"] = None,
):
client = self.get_azure_openai_client(
litellm_params=litellm_params,
@@ -69,4 +73,5 @@ class AzureOpenAIO1ChatCompletion(BaseAzureLLM, OpenAIChatCompletion):
organization=organization,
custom_llm_provider=custom_llm_provider,
drop_params=drop_params,
+ shared_session=shared_session,
)
diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py
index c7f7acf331d..63c366c9480 100644
--- a/litellm/llms/bedrock/common_utils.py
+++ b/litellm/llms/bedrock/common_utils.py
@@ -774,7 +774,7 @@ class CommonBatchFilesUtils:
Returns:
Unique job name (≤ 63 characters for Bedrock compatibility)
"""
- import fastuuid as uuid
+ from litellm._uuid import uuid
unique_id = str(uuid.uuid4())[:8]
# Format: {prefix}-batch-{model}-{uuid}
# Example: litellm-batch-claude-266c398e
diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py
index 36b543086f5..3a67d5127b7 100644
--- a/litellm/llms/custom_httpx/http_handler.py
+++ b/litellm/llms/custom_httpx/http_handler.py
@@ -167,6 +167,7 @@ class AsyncHTTPHandler:
concurrent_limit=1000,
client_alias: Optional[str] = None, # name for client in logs
ssl_verify: Optional[VerifyTypes] = None,
+ shared_session: Optional["ClientSession"] = None,
):
self.timeout = timeout
self.event_hooks = event_hooks
@@ -175,6 +176,7 @@ class AsyncHTTPHandler:
concurrent_limit=concurrent_limit,
event_hooks=event_hooks,
ssl_verify=ssl_verify,
+ shared_session=shared_session,
)
self.client_alias = client_alias
@@ -184,6 +186,7 @@ class AsyncHTTPHandler:
concurrent_limit: int,
event_hooks: Optional[Mapping[str, List[Callable[..., Any]]]],
ssl_verify: Optional[VerifyTypes] = None,
+ shared_session: Optional["ClientSession"] = None,
) -> httpx.AsyncClient:
# Get unified SSL configuration
ssl_config = get_ssl_configuration(ssl_verify)
@@ -199,6 +202,7 @@ class AsyncHTTPHandler:
transport = AsyncHTTPHandler._create_async_transport(
ssl_context=ssl_config if isinstance(ssl_config, ssl.SSLContext) else None,
ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
+ shared_session=shared_session,
)
return httpx.AsyncClient(
@@ -260,7 +264,6 @@ class AsyncHTTPHandler:
files: Optional[RequestFiles] = None,
content: Any = None,
):
-
start_time = time.time()
try:
if timeout is None:
@@ -523,7 +526,9 @@ class AsyncHTTPHandler:
@staticmethod
def _create_async_transport(
- ssl_context: Optional[ssl.SSLContext] = None, ssl_verify: Optional[bool] = None
+ ssl_context: Optional[ssl.SSLContext] = None,
+ ssl_verify: Optional[bool] = None,
+ shared_session: Optional["ClientSession"] = None,
) -> Optional[Union[LiteLLMAiohttpTransport, AsyncHTTPTransport]]:
"""
- Creates a transport for httpx.AsyncClient
@@ -544,7 +549,9 @@ class AsyncHTTPHandler:
#########################################################
if AsyncHTTPHandler._should_use_aiohttp_transport():
return AsyncHTTPHandler._create_aiohttp_transport(
- ssl_context=ssl_context, ssl_verify=ssl_verify
+ ssl_context=ssl_context,
+ ssl_verify=ssl_verify,
+ shared_session=shared_session,
)
#########################################################
@@ -612,6 +619,7 @@ class AsyncHTTPHandler:
def _create_aiohttp_transport(
ssl_verify: Optional[bool] = None,
ssl_context: Optional[ssl.SSLContext] = None,
+ shared_session: Optional["ClientSession"] = None,
) -> LiteLLMAiohttpTransport:
"""
Creates an AiohttpTransport with RequestNotRead error handling
@@ -635,6 +643,18 @@ class AsyncHTTPHandler:
trust_env = True
verbose_logger.debug("Creating AiohttpTransport...")
+
+ # Use shared session if provided and valid
+ if shared_session is not None and not shared_session.closed:
+ verbose_logger.debug(
+ f"SHARED SESSION: Reusing existing ClientSession (ID: {id(shared_session)})"
+ )
+ return LiteLLMAiohttpTransport(client=shared_session)
+
+ # Create new session only if none provided or existing one is invalid
+ verbose_logger.debug(
+ "NEW SESSION: Creating new ClientSession (no shared session provided)"
+ )
return LiteLLMAiohttpTransport(
client=lambda: ClientSession(
connector=TCPConnector(**connector_kwargs),
@@ -915,12 +935,13 @@ class HTTPHandler:
if litellm.force_ipv4:
return HTTPTransport(local_address="0.0.0.0")
else:
- return None
+ return getattr(litellm, 'sync_transport', None)
def get_async_httpx_client(
llm_provider: Union[LlmProviders, httpxSpecialProvider],
params: Optional[dict] = None,
+ shared_session: Optional["ClientSession"] = None,
) -> AsyncHTTPHandler:
"""
Retrieves the async HTTP client from the cache
@@ -942,10 +963,12 @@ def get_async_httpx_client(
return _cached_client
if params is not None:
+ params["shared_session"] = shared_session
_new_client = AsyncHTTPHandler(**params)
else:
_new_client = AsyncHTTPHandler(
- timeout=httpx.Timeout(timeout=600.0, connect=5.0)
+ timeout=httpx.Timeout(timeout=600.0, connect=5.0),
+ shared_session=shared_session,
)
litellm.in_memory_llm_clients_cache.set_cache(
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index dc64fea1a33..f8a92c2ac96 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -88,6 +88,7 @@ from litellm.utils import (
)
if TYPE_CHECKING:
+ from aiohttp import ClientSession
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
@@ -236,11 +237,16 @@ class BaseLLMHTTPHandler:
client: Optional[AsyncHTTPHandler] = None,
json_mode: bool = False,
signed_json_body: Optional[bytes] = None,
+ shared_session: Optional["ClientSession"] = None,
):
if client is None:
+ verbose_logger.debug(
+ f"Creating HTTP client with shared_session: {id(shared_session) if shared_session else None}"
+ )
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
params={"ssl_verify": litellm_params.get("ssl_verify", None)},
+ shared_session=shared_session,
)
else:
async_httpx_client = client
@@ -290,6 +296,7 @@ class BaseLLMHTTPHandler:
headers: Optional[Dict[str, Any]] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
provider_config: Optional[BaseConfig] = None,
+ shared_session: Optional["ClientSession"] = None,
):
json_mode: bool = optional_params.pop("json_mode", False)
extra_body: Optional[dict] = optional_params.pop("extra_body", None)
@@ -469,7 +476,7 @@ class BaseLLMHTTPHandler:
if client is None or not isinstance(client, HTTPHandler):
sync_httpx_client = _get_httpx_client(
- params={"ssl_verify": litellm_params.get("ssl_verify", None)}
+ params={"ssl_verify": litellm_params.get("ssl_verify", None)},
)
else:
sync_httpx_client = client
@@ -2283,7 +2290,7 @@ class BaseLLMHTTPHandler:
e=e,
provider_config=provider_config,
)
-
+
# Store the upload URL in litellm_params for the transformation method
litellm_params_with_url = dict(litellm_params)
litellm_params_with_url["upload_url"] = api_base
@@ -2574,11 +2581,11 @@ class BaseLLMHTTPHandler:
"url": transformed_request["url"],
"headers": transformed_request["headers"],
}
-
+
# Only add data for non-GET requests
if method != "get" and transformed_request.get("data") is not None:
request_kwargs["data"] = transformed_request["data"]
-
+
batch_response = getattr(sync_httpx_client, method)(**request_kwargs)
elif isinstance(transformed_request, dict) and api_base:
# For other providers that use JSON requests
@@ -2743,12 +2750,14 @@ class BaseLLMHTTPHandler:
"url": transformed_request["url"],
"headers": transformed_request["headers"],
}
-
+
# Only add data for non-GET requests
if method != "get" and transformed_request.get("data") is not None:
request_kwargs["data"] = transformed_request["data"]
-
- batch_response = await getattr(async_httpx_client, method)(**request_kwargs)
+
+ batch_response = await getattr(async_httpx_client, method)(
+ **request_kwargs
+ )
elif isinstance(transformed_request, dict) and api_base:
# For other providers that use JSON requests
batch_response = await async_httpx_client.get(
diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py
index aa670df0531..00197a67e95 100644
--- a/litellm/llms/openai/common_utils.py
+++ b/litellm/llms/openai/common_utils.py
@@ -5,12 +5,15 @@ Common helpers / utils across al OpenAI endpoints
import hashlib
import json
import ssl
-from typing import Any, Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, TYPE_CHECKING, Union
import httpx
import openai
from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
+if TYPE_CHECKING:
+ from aiohttp import ClientSession
+
import litellm
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
@@ -194,7 +197,9 @@ class BaseOpenAILLM:
return param_names
@staticmethod
- def _get_async_http_client() -> Optional[httpx.AsyncClient]:
+ def _get_async_http_client(
+ shared_session: Optional["ClientSession"] = None,
+ ) -> Optional[httpx.AsyncClient]:
if litellm.aclient_session is not None:
return litellm.aclient_session
@@ -205,8 +210,11 @@ class BaseOpenAILLM:
limits=httpx.Limits(max_connections=1000, max_keepalive_connections=100),
verify=ssl_config,
transport=AsyncHTTPHandler._create_async_transport(
- ssl_context=ssl_config if isinstance(ssl_config, ssl.SSLContext) else None,
+ ssl_context=ssl_config
+ if isinstance(ssl_config, ssl.SSLContext)
+ else None,
ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
+ shared_session=shared_session,
),
follow_redirects=True,
)
@@ -215,10 +223,10 @@ class BaseOpenAILLM:
def _get_sync_http_client() -> Optional[httpx.Client]:
if litellm.client_session is not None:
return litellm.client_session
-
+
# Get unified SSL configuration
ssl_config = get_ssl_configuration()
-
+
return httpx.Client(
limits=httpx.Limits(max_connections=1000, max_keepalive_connections=100),
verify=ssl_config,
diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py
index 1f3cf24457d..3347e533242 100644
--- a/litellm/llms/openai/openai.py
+++ b/litellm/llms/openai/openai.py
@@ -10,12 +10,16 @@ from typing import (
List,
Literal,
Optional,
+ TYPE_CHECKING,
Union,
cast,
)
from urllib.parse import urlparse
import httpx
+
+if TYPE_CHECKING:
+ from aiohttp import ClientSession
import openai
from openai import AsyncOpenAI, OpenAI
from openai.types.beta.assistant_deleted import AssistantDeleted
@@ -355,6 +359,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
max_retries: Optional[int] = DEFAULT_MAX_RETRIES,
organization: Optional[str] = None,
client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
+ shared_session: Optional["ClientSession"] = None,
) -> Optional[Union[OpenAI, AsyncOpenAI]]:
client_initialization_params: Dict = locals()
if client is None:
@@ -379,7 +384,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
_new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI(
api_key=api_key,
base_url=api_base,
- http_client=OpenAIChatCompletion._get_async_http_client(),
+ http_client=OpenAIChatCompletion._get_async_http_client(
+ shared_session=shared_session
+ ),
timeout=timeout,
max_retries=max_retries,
organization=organization,
@@ -522,8 +529,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
organization: Optional[str] = None,
custom_llm_provider: Optional[str] = None,
drop_params: Optional[bool] = None,
+ shared_session: Optional["ClientSession"] = None,
):
- super().completion()
+ super().completion(shared_session=shared_session)
try:
fake_stream: bool = False
inference_params = optional_params.copy()
@@ -606,6 +614,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
organization=organization,
drop_params=drop_params,
fake_stream=fake_stream,
+ shared_session=shared_session,
)
data = provider_config.transform_request(
@@ -771,6 +780,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
drop_params: Optional[bool] = None,
stream_options: Optional[dict] = None,
fake_stream: bool = False,
+ shared_session: Optional["ClientSession"] = None,
):
response = None
data = await provider_config.async_transform_request(
@@ -793,6 +803,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
max_retries=max_retries,
organization=organization,
client=client,
+ shared_session=shared_session,
)
## LOGGING
diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py
index c59e3bb24e8..ccaf28e5906 100644
--- a/litellm/llms/vertex_ai/gemini/transformation.py
+++ b/litellm/llms/vertex_ai/gemini/transformation.py
@@ -537,7 +537,11 @@ def sync_transform_request_body(
logging_obj=logging_obj,
)
else: # [TODO] implement context caching for gemini as well
- cached_content = optional_params.pop("cached_content", None)
+ cached_content = None
+ if "cached_content" in optional_params:
+ cached_content = optional_params.pop("cached_content")
+ elif "cachedContent" in optional_params:
+ cached_content = optional_params.pop("cachedContent")
return _transform_request_body(
messages=messages,
@@ -584,7 +588,11 @@ async def async_transform_request_body(
logging_obj=logging_obj,
)
else: # [TODO] implement context caching for gemini as well
- cached_content = optional_params.pop("cached_content", None)
+ cached_content = None
+ if "cached_content" in optional_params:
+ cached_content = optional_params.pop("cached_content")
+ elif "cachedContent" in optional_params:
+ cached_content = optional_params.pop("cachedContent")
return _transform_request_body(
messages=messages,
@@ -649,5 +657,3 @@ def _transform_system_message(
return SystemInstructions(parts=system_content_blocks), messages
return None, messages
-
-
diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py
index 0f0bc776cc9..6d194d41add 100644
--- a/litellm/llms/vertex_ai/vertex_llm_base.py
+++ b/litellm/llms/vertex_ai/vertex_llm_base.py
@@ -271,17 +271,11 @@ class VertexBase:
def is_using_v1beta1_features(self, optional_params: dict) -> bool:
"""
- VertexAI only supports ContextCaching on v1beta1
-
use this helper to decide if request should be sent to v1 or v1beta1
- Returns v1beta1 if context caching is enabled
- Returns v1 in all other cases
+ Returns true if any beta feature is enabled
+ Returns false in all other cases
"""
- if "cached_content" in optional_params:
- return True
- if "CachedContent" in optional_params:
- return True
return False
def _check_custom_proxy(
diff --git a/litellm/main.py b/litellm/main.py
index 5493c7e34e3..50de09c3dbf 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -36,8 +36,12 @@ from typing import (
Union,
cast,
get_args,
+ TYPE_CHECKING,
)
+if TYPE_CHECKING:
+ from aiohttp import ClientSession
+
import dotenv
import httpx
import openai
@@ -374,6 +378,8 @@ async def acompletion(
# Optional liteLLM function params
thinking: Optional[AnthropicThinkingParam] = None,
web_search_options: Optional[OpenAIWebSearchOptions] = None,
+ # Session management
+ shared_session: Optional["ClientSession"] = None,
**kwargs,
) -> Union[ModelResponse, CustomStreamWrapper]:
"""
@@ -466,6 +472,16 @@ async def acompletion(
#########################################################
#########################################################
+ # Log shared session usage
+ if shared_session is not None:
+ verbose_logger.debug(
+ f"🔄 SHARED SESSION: acompletion called with shared_session (ID: {id(shared_session)})"
+ )
+ else:
+ verbose_logger.debug(
+ "🔄 NO SHARED SESSION: acompletion called without shared_session"
+ )
+
# Adjusted to use explicit arguments instead of *args and **kwargs
completion_kwargs = {
"model": model,
@@ -506,6 +522,7 @@ async def acompletion(
"acompletion": True, # assuming this is a required parameter
"thinking": thinking,
"web_search_options": web_search_options,
+ "shared_session": shared_session,
}
if custom_llm_provider is None:
_, custom_llm_provider, _, _ = get_llm_provider(
@@ -930,6 +947,8 @@ def completion( # type: ignore # noqa: PLR0915
model_list: Optional[list] = None, # pass in a list of api_base,keys, etc.
# Optional liteLLM function params
thinking: Optional[AnthropicThinkingParam] = None,
+ # Session management
+ shared_session: Optional["ClientSession"] = None,
**kwargs,
) -> Union[ModelResponse, CustomStreamWrapper]:
"""
@@ -1004,15 +1023,7 @@ def completion( # type: ignore # noqa: PLR0915
provider_specific_header = cast(
Optional[ProviderSpecificHeader], kwargs.get("provider_specific_header", None)
)
- # Properly merge headers with priority: request headers > extra_headers > global litellm.headers
- headers = {}
- if litellm.headers is not None and isinstance(litellm.headers, dict):
- headers.update(litellm.headers)
- if extra_headers is not None and isinstance(extra_headers, dict):
- headers.update(extra_headers)
- request_headers = kwargs.get("headers", None)
- if request_headers is not None and isinstance(request_headers, dict):
- headers.update(request_headers)
+ headers = kwargs.get("headers", None) or extra_headers
ensure_alternating_roles: Optional[bool] = kwargs.get(
"ensure_alternating_roles", None
@@ -1023,6 +1034,10 @@ def completion( # type: ignore # noqa: PLR0915
assistant_continue_message: Optional[ChatCompletionAssistantMessage] = kwargs.get(
"assistant_continue_message", None
)
+ if headers is None:
+ headers = {}
+ if extra_headers is not None:
+ headers.update(extra_headers)
num_retries = kwargs.get(
"num_retries", None
) ## alt. param for 'max_retries'. Use this to pass retries w/ instructor.
@@ -1079,6 +1094,7 @@ def completion( # type: ignore # noqa: PLR0915
prompt_id=prompt_id, non_default_params=non_default_params
)
):
+
(
model,
messages,
@@ -1431,7 +1447,8 @@ def completion( # type: ignore # noqa: PLR0915
"azure_ad_token_provider", None
)
- # Use the consolidated headers that were already merged at the top of the function
+ headers = headers or litellm.headers
+
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
if max_retries is not None:
@@ -1598,6 +1615,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
timeout=timeout, # type: ignore
client=client,
custom_llm_provider=custom_llm_provider,
@@ -1644,6 +1662,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
timeout=timeout, # type: ignore
client=client, # pass AsyncOpenAI, OpenAI client
custom_llm_provider=custom_llm_provider,
@@ -1696,7 +1715,8 @@ def completion( # type: ignore # noqa: PLR0915
or get_secret("OPENAI_API_KEY")
)
- # Use the consolidated headers that were already merged at the top of the function
+ headers = headers or litellm.headers
+
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
@@ -1772,6 +1792,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
timeout=timeout, # type: ignore
client=client,
custom_llm_provider=custom_llm_provider,
@@ -1801,6 +1822,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
@@ -1831,6 +1853,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
timeout=timeout, # type: ignore
client=client,
custom_llm_provider=custom_llm_provider,
@@ -1882,6 +1905,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
custom_llm_provider=custom_llm_provider,
timeout=timeout,
headers=headers,
@@ -1955,6 +1979,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
timeout=timeout,
client=client,
custom_llm_provider=custom_llm_provider,
@@ -2034,6 +2059,7 @@ def completion( # type: ignore # noqa: PLR0915
try:
if use_base_llm_http_handler:
+
response = base_llm_http_handler.completion(
model=model,
messages=messages,
@@ -2045,6 +2071,7 @@ def completion( # type: ignore # noqa: PLR0915
optional_params=optional_params,
timeout=timeout,
litellm_params=litellm_params,
+ shared_session=shared_session,
acompletion=acompletion,
stream=stream,
api_key=api_key,
@@ -2071,6 +2098,7 @@ def completion( # type: ignore # noqa: PLR0915
client=client, # pass AsyncOpenAI, OpenAI client
organization=organization,
custom_llm_provider=custom_llm_provider,
+ shared_session=shared_session,
)
except Exception as e:
## LOGGING - log the original exception returned
@@ -2111,6 +2139,7 @@ def completion( # type: ignore # noqa: PLR0915
optional_params=optional_params,
timeout=timeout,
litellm_params=litellm_params,
+ shared_session=shared_session,
acompletion=acompletion,
stream=stream,
api_key=api_key,
@@ -2198,6 +2227,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
custom_llm_provider="clarifai",
timeout=timeout,
headers=headers,
@@ -2243,6 +2273,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
custom_llm_provider="anthropic_text",
timeout=timeout,
headers=headers,
@@ -2412,8 +2443,12 @@ def completion( # type: ignore # noqa: PLR0915
or "https://api.cohere.ai/v1/chat"
)
- # Use the consolidated headers that were already merged at the top of the function
- # No need for additional merging here as it's already done
+ headers = headers or litellm.headers or {}
+ if headers is None:
+ headers = {}
+
+ if extra_headers is not None:
+ headers.update(extra_headers)
response = base_llm_http_handler.completion(
model=model,
@@ -2424,6 +2459,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
custom_llm_provider="cohere_chat",
timeout=timeout,
headers=headers,
@@ -2509,10 +2545,15 @@ def completion( # type: ignore # noqa: PLR0915
)
elif custom_llm_provider == "compactifai":
api_key = (
- api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key
+ api_key
+ or get_secret_str("COMPACTIFAI_API_KEY")
+ or litellm.api_key
)
- api_base = api_base or "https://api.compactif.ai/v1"
+ api_base = (
+ api_base
+ or "https://api.compactif.ai/v1"
+ )
## COMPLETION CALL
response = base_llm_http_handler.completion(
@@ -2686,6 +2727,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
custom_llm_provider="openrouter",
timeout=timeout,
headers=headers,
@@ -2748,6 +2790,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
custom_llm_provider="vercel_ai_gateway",
timeout=timeout,
headers=headers,
@@ -3098,9 +3141,9 @@ def completion( # type: ignore # noqa: PLR0915
"aws_region_name" not in optional_params
or optional_params["aws_region_name"] is None
):
- optional_params[
- "aws_region_name"
- ] = aws_bedrock_client.meta.region_name
+ optional_params["aws_region_name"] = (
+ aws_bedrock_client.meta.region_name
+ )
bedrock_route = BedrockModelInfo.get_bedrock_route(model)
if bedrock_route == "converse":
@@ -3234,6 +3277,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
custom_llm_provider="watsonx_text",
timeout=timeout,
headers=headers,
@@ -3287,6 +3331,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
custom_llm_provider="ollama",
timeout=timeout,
headers=headers,
@@ -3320,6 +3365,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
custom_llm_provider="ollama_chat",
timeout=timeout,
headers=headers,
@@ -3340,6 +3386,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
custom_llm_provider=custom_llm_provider,
timeout=timeout,
headers=headers,
@@ -3372,6 +3419,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
custom_llm_provider="cloudflare",
timeout=timeout,
headers=headers,
@@ -3425,6 +3473,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
timeout=timeout, # type: ignore
client=client,
custom_llm_provider=custom_llm_provider,
@@ -3442,6 +3491,7 @@ def completion( # type: ignore # noqa: PLR0915
)
raise e
elif custom_llm_provider == "gradient_ai":
+
api_base = litellm.api_base or api_base
response = base_llm_http_handler.completion(
model=model,
@@ -3452,6 +3502,7 @@ def completion( # type: ignore # noqa: PLR0915
model_response=model_response,
optional_params=optional_params,
litellm_params=litellm_params,
+ shared_session=shared_session,
custom_llm_provider="gradient_ai",
timeout=timeout,
headers=headers,
@@ -3605,7 +3656,7 @@ def completion( # type: ignore # noqa: PLR0915
async_fn=acompletion, stream=stream, custom_llm=custom_handler
)
- headers = headers or litellm.headers
+ headers = headers or litellm.headers or {}
## CALL FUNCTION
response = handler_fn(
@@ -3801,7 +3852,7 @@ def embedding(
*,
aembedding: Literal[True],
**kwargs,
-) -> Coroutine[Any, Any, EmbeddingResponse]:
+) -> Coroutine[Any, Any, EmbeddingResponse]:
...
@@ -3827,7 +3878,7 @@ def embedding(
*,
aembedding: Literal[False] = False,
**kwargs,
-) -> EmbeddingResponse:
+) -> EmbeddingResponse:
...
# fmt: on
@@ -4138,8 +4189,10 @@ def embedding( # noqa: PLR0915
or litellm.api_key
)
- # Use the consolidated headers that were already merged at the top of the function
- # No need for additional merging here as it's already done
+ if extra_headers is not None and isinstance(extra_headers, dict):
+ headers = extra_headers
+ else:
+ headers = {}
response = base_llm_http_handler.embedding(
model=model,
@@ -5099,9 +5152,9 @@ def adapter_completion(
new_kwargs = translation_obj.translate_completion_input_params(kwargs=kwargs)
response: Union[ModelResponse, CustomStreamWrapper] = completion(**new_kwargs) # type: ignore
- translated_response: Optional[
- Union[BaseModel, AdapterCompletionStreamWrapper]
- ] = None
+ translated_response: Optional[Union[BaseModel, AdapterCompletionStreamWrapper]] = (
+ None
+ )
if isinstance(response, ModelResponse):
translated_response = translation_obj.translate_completion_output_params(
response=response
@@ -6089,9 +6142,9 @@ def stream_chunk_builder( # noqa: PLR0915
]
if len(content_chunks) > 0:
- response["choices"][0]["message"][
- "content"
- ] = processor.get_combined_content(content_chunks)
+ response["choices"][0]["message"]["content"] = (
+ processor.get_combined_content(content_chunks)
+ )
thinking_blocks = [
chunk
@@ -6102,9 +6155,9 @@ def stream_chunk_builder( # noqa: PLR0915
]
if len(thinking_blocks) > 0:
- response["choices"][0]["message"][
- "thinking_blocks"
- ] = processor.get_combined_thinking_content(thinking_blocks)
+ response["choices"][0]["message"]["thinking_blocks"] = (
+ processor.get_combined_thinking_content(thinking_blocks)
+ )
reasoning_chunks = [
chunk
@@ -6115,9 +6168,9 @@ def stream_chunk_builder( # noqa: PLR0915
]
if len(reasoning_chunks) > 0:
- response["choices"][0]["message"][
- "reasoning_content"
- ] = processor.get_combined_reasoning_content(reasoning_chunks)
+ response["choices"][0]["message"]["reasoning_content"] = (
+ processor.get_combined_reasoning_content(reasoning_chunks)
+ )
audio_chunks = [
chunk
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 578523abff4..b0261d1bd43 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -560,10 +560,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -824,10 +828,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -4787,10 +4795,14 @@
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "anthropic",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -7191,6 +7203,18 @@
"supports_prompt_caching": true,
"supports_tool_choice": true
},
+ "deepseek.v3-v1:0": {
+ "input_cost_per_token": 5.8e-07,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 163840,
+ "max_output_tokens": 81920,
+ "max_tokens": 163840,
+ "mode": "chat",
+ "output_cost_per_token": 1.68e-06,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
"dolphin": {
"input_cost_per_token": 5e-07,
"litellm_provider": "nlp_cloud",
@@ -7547,10 +7571,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -12332,6 +12360,7 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
+ "supports_service_tier": true,
"supports_vision": true
},
"gpt-5-chat": {
@@ -12479,6 +12508,7 @@
"cache_read_input_token_cost_flex": 2.5e-09,
"input_cost_per_token": 5e-08,
"input_cost_per_token_flex": 2.5e-08,
+ "input_cost_per_token_priority": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_output_tokens": 128000,
@@ -15452,7 +15482,7 @@
},
"o4-mini": {
"cache_read_input_token_cost": 2.75e-07,
- "cache_read_input_token_cost_flex": 1.38e-07,
+ "cache_read_input_token_cost_flex": 1.375e-07,
"cache_read_input_token_cost_priority": 5e-07,
"input_cost_per_token": 1.1e-06,
"input_cost_per_token_flex": 5.5e-07,
@@ -16155,10 +16185,12 @@
"openrouter/anthropic/claude-sonnet-4": {
"input_cost_per_image": 0.0048,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
"litellm_provider": "openrouter",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
@@ -17500,6 +17532,54 @@
"mode": "chat",
"output_cost_per_token": 2.8e-07
},
+ "qwen.qwen3-coder-480b-a35b-v1:0": {
+ "input_cost_per_token": 2.2e-07,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 262000,
+ "max_output_tokens": 65536,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 1.8e-06,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
+ "qwen.qwen3-235b-a22b-2507-v1:0": {
+ "input_cost_per_token": 2.2e-07,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 131072,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 8.8e-07,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
+ "qwen.qwen3-coder-30b-a3b-v1:0": {
+ "input_cost_per_token": 1.5e-07,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 131072,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 6.0e-07,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
+ "qwen.qwen3-32b-v1:0": {
+ "input_cost_per_token": 1.5e-07,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 16384,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 6.0e-07,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
"recraft/recraftv2": {
"litellm_provider": "recraft",
"mode": "image_generation",
@@ -19018,10 +19098,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -20354,10 +20438,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "vertex_ai-anthropic_models",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -20380,10 +20468,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "vertex_ai-anthropic_models",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -21187,6 +21279,30 @@
"/v1/audio/transcriptions"
]
},
+ "vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": {
+ "input_cost_per_token": 1.5e-07,
+ "litellm_provider": "vertex_ai-qwen_models",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+ "supports_function_calling": true,
+ "supports_tool_choice": true
+ },
+ "vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": {
+ "input_cost_per_token": 1.5e-07,
+ "litellm_provider": "vertex_ai-qwen_models",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+ "supports_function_calling": true,
+ "supports_tool_choice": true
+ },
"xai/grok-2": {
"input_cost_per_token": 2e-06,
"litellm_provider": "xai",
@@ -21439,6 +21555,35 @@
"supports_tool_choice": true,
"supports_web_search": true
},
+ "xai/grok-4-fast-reasoning": {
+ "litellm_provider": "xai",
+ "max_input_tokens": 2e6,
+ "max_output_tokens": 2e6,
+ "max_tokens": 2e6,
+ "mode": "chat",
+ "input_cost_per_token": 0.2e-06,
+ "output_cost_per_token": 0.5e-06,
+ "cache_read_input_token_cost": 0.05e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_web_search": true
+ },
+ "xai/grok-4-fast-non-reasoning": {
+ "litellm_provider": "xai",
+ "max_input_tokens": 2e6,
+ "max_output_tokens": 2e6,
+ "cache_read_input_token_cost": 0.05e-06,
+ "max_tokens": 2e6,
+ "mode": "chat",
+ "input_cost_per_token": 0.2e-06,
+ "output_cost_per_token": 0.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_web_search": true
+ },
"xai/grok-4-0709": {
"input_cost_per_token": 3e-06,
"litellm_provider": "xai",
diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
index 1e7840d95b3..ab5d1b10bf3 100644
--- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
+++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
@@ -421,7 +421,6 @@ class MCPServerManager:
self,
server: MCPServer,
mcp_auth_header: Optional[str] = None,
- add_prefix: bool = True,
) -> List[MCPTool]:
"""
Helper method to get tools from a single MCP server with prefixed names.
@@ -446,11 +445,9 @@ class MCPServerManager:
tools = await self._fetch_tools_with_timeout(client, server.name)
- prefixed_or_original_tools = self._create_prefixed_tools(
- tools, server, add_prefix=add_prefix
- )
+ prefixed_tools = self._create_prefixed_tools(tools, server)
- return prefixed_or_original_tools
+ return prefixed_tools
except Exception as e:
verbose_logger.warning(
@@ -519,7 +516,7 @@ class MCPServerManager:
return []
def _create_prefixed_tools(
- self, tools: List[MCPTool], server: MCPServer, add_prefix: bool = True
+ self, tools: List[MCPTool], server: MCPServer
) -> List[MCPTool]:
"""
Create prefixed tools and update tool mapping.
@@ -537,16 +534,14 @@ class MCPServerManager:
for tool in tools:
prefixed_name = add_server_prefix_to_tool_name(tool.name, prefix)
- name_to_use = prefixed_name if add_prefix else tool.name
-
- tool_obj = MCPTool(
- name=name_to_use,
+ prefixed_tool = MCPTool(
+ name=prefixed_name,
description=tool.description,
inputSchema=tool.inputSchema,
)
- prefixed_tools.append(tool_obj)
+ prefixed_tools.append(prefixed_tool)
- # Update tool to server mapping for resolution (support both forms)
+ # Update tool to server mapping with both original and prefixed names
self.tool_name_to_mcp_server_name_mapping[tool.name] = prefix
self.tool_name_to_mcp_server_name_mapping[prefixed_name] = prefix
diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py
index 399b79b4f7c..2a9174717d1 100644
--- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py
+++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py
@@ -73,7 +73,6 @@ if MCP_AVAILABLE:
tools = await global_mcp_server_manager._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
- add_prefix=False,
)
return _create_tool_response_objects(tools, server.mcp_info)
diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py
index 8558229cf0b..2cf91c84dcc 100644
--- a/litellm/proxy/_experimental/mcp_server/server.py
+++ b/litellm/proxy/_experimental/mcp_server/server.py
@@ -384,9 +384,6 @@ if MCP_AVAILABLE:
allowed_mcp_servers=allowed_mcp_servers,
)
- # Decide whether to add prefix based on number of allowed servers
- add_prefix = not (len(allowed_mcp_servers) == 1)
-
# Get tools from each allowed server
all_tools = []
for server_id in allowed_mcp_servers:
@@ -409,7 +406,6 @@ if MCP_AVAILABLE:
tools = await global_mcp_server_manager._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
- add_prefix=add_prefix,
)
all_tools.extend(tools)
verbose_logger.debug(
@@ -641,35 +637,27 @@ if MCP_AVAILABLE:
# Server names can contain slashes (e.g., "custom_solutions/user_123")
mcp_path_match = re.match(r"^/mcp/([^?#]+)(?:\?.*)?(?:#.*)?$", path)
if mcp_path_match:
- mcp_servers_str = mcp_path_match.group(1)
- optional_path = mcp_path_match.group(2)
-
- if mcp_servers_str:
- # First, try to split by comma for comma-separated lists
- if "," in mcp_servers_str:
- # For comma-separated lists, we need to handle the case where the last item
- # might include the path (e.g., "zapier,group1/tools" -> ["zapier", "group1/tools"])
- parts = [s.strip() for s in mcp_servers_str.split(",") if s.strip()]
-
- # If there's an optional path AND the last part contains a slash that matches the optional path,
- # remove the path portion from the last server name
- if optional_path and len(parts) > 0 and "/" in parts[-1]:
- last_part = parts[-1]
- # Check if the last part ends with the optional path
- if optional_path and last_part.endswith(
- optional_path.lstrip("/")
- ):
- # Remove the path portion from the last server name
- parts[-1] = last_part[: -len(optional_path.lstrip("/"))]
-
- mcp_servers_from_path = parts
+ servers_and_path = mcp_path_match.group(1)
+
+ if servers_and_path:
+ # Check if it contains commas (comma-separated servers)
+ if ',' in servers_and_path:
+ # For comma-separated, look for a path at the end
+ # Common patterns: /tools, /chat/completions, etc.
+ path_match = re.search(r'/([^/,]+(?:/[^/,]+)*)$', servers_and_path)
+ if path_match:
+ # Path found at the end, remove it from servers
+ path_part = '/' + path_match.group(1)
+ servers_part = servers_and_path[:-len(path_part)]
+ mcp_servers_from_path = [s.strip() for s in servers_part.split(',') if s.strip()]
+ else:
+ # No path, just comma-separated servers
+ mcp_servers_from_path = [s.strip() for s in servers_and_path.split(',') if s.strip()]
else:
- # For single server, it might be just a name or contain slashes
- # We need to determine where the server name ends and the path begins
- # This is tricky - let's use the original logic but handle comma cases differently
- single_server_match = re.match(
- r"^([^/]+(?:/[^/]+)?)(?:/.*)?$", mcp_servers_str
- )
+ # Single server case - use regex approach for server/path separation
+ # This handles cases like "custom_solutions/user_123/chat/completions"
+ # where we want to extract "custom_solutions/user_123" as the server name
+ single_server_match = re.match(r"^([^/]+(?:/[^/]+)?)(?:/.*)?$", servers_and_path)
if single_server_match:
server_name = single_server_match.group(1)
mcp_servers_from_path = [server_name]
diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py
index 8a138b2a808..012876db481 100644
--- a/litellm/proxy/auth/auth_checks.py
+++ b/litellm/proxy/auth/auth_checks.py
@@ -469,7 +469,6 @@ async def get_end_user_object(
# check if in cache
cached_user_obj = await user_api_key_cache.async_get_cache(key=_key)
if cached_user_obj is not None:
- # Convert cached dict to LiteLLM_EndUserTable instance
return_obj = LiteLLM_EndUserTable(**cached_user_obj)
check_in_budget(end_user_obj=return_obj)
return return_obj
@@ -527,10 +526,7 @@ async def get_team_membership(
# check if in cache
cached_membership_obj = await user_api_key_cache.async_get_cache(key=_key)
if cached_membership_obj is not None:
- if isinstance(cached_membership_obj, dict):
- return LiteLLM_TeamMembership(**cached_membership_obj)
- elif isinstance(cached_membership_obj, LiteLLM_TeamMembership):
- return cached_membership_obj
+ return LiteLLM_TeamMembership(**cached_membership_obj)
# else, check db
try:
@@ -542,8 +538,8 @@ async def get_team_membership(
if response is None:
return None
- # save the team membership object to cache
- await user_api_key_cache.async_set_cache(key=_key, value=response)
+ # save the team membership object to cache (store as dict)
+ await user_api_key_cache.async_set_cache(key=_key, value=response.dict())
_response = LiteLLM_TeamMembership(**response.dict())
@@ -819,8 +815,9 @@ async def _cache_management_object(
user_api_key_cache: DualCache,
proxy_logging_obj: Optional[ProxyLogging],
):
+
await user_api_key_cache.async_set_cache(
- key=key, value=value, ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
+ key=key, value=value, ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
)
diff --git a/litellm/proxy/client/chat.py b/litellm/proxy/client/chat.py
index bf5ddbe85d9..91fc33002b4 100644
--- a/litellm/proxy/client/chat.py
+++ b/litellm/proxy/client/chat.py
@@ -1,5 +1,8 @@
+import json
+from typing import Any, Dict, Iterator, List, Optional, Union
+
import requests
-from typing import List, Dict, Any, Optional, Union
+
from .exceptions import UnauthorizedError
@@ -99,3 +102,91 @@ class ChatClient:
if e.response.status_code == 401:
raise UnauthorizedError(e)
raise
+
+ def completions_stream(
+ self,
+ model: str,
+ messages: List[Dict[str, str]],
+ temperature: Optional[float] = None,
+ top_p: Optional[float] = None,
+ n: Optional[int] = None,
+ max_tokens: Optional[int] = None,
+ presence_penalty: Optional[float] = None,
+ frequency_penalty: Optional[float] = None,
+ user: Optional[str] = None,
+ ) -> Iterator[Dict[str, Any]]:
+ """
+ Create a streaming chat completion.
+
+ Args:
+ model (str): The model to use for completion
+ messages (List[Dict[str, str]]): The messages to generate a completion for
+ temperature (Optional[float]): Sampling temperature between 0 and 2
+ top_p (Optional[float]): Nucleus sampling parameter between 0 and 1
+ n (Optional[int]): Number of completions to generate
+ max_tokens (Optional[int]): Maximum number of tokens to generate
+ presence_penalty (Optional[float]): Presence penalty between -2.0 and 2.0
+ frequency_penalty (Optional[float]): Frequency penalty between -2.0 and 2.0
+ user (Optional[str]): Unique identifier for the end user
+
+ Yields:
+ Dict[str, Any]: Streaming response chunks from the server
+
+ Raises:
+ UnauthorizedError: If the request fails with a 401 status code
+ requests.exceptions.RequestException: If the request fails with any other error
+ """
+ url = f"{self._base_url}/chat/completions"
+
+ # Build request data with required fields
+ data: Dict[str, Any] = {
+ "model": model,
+ "messages": messages,
+ "stream": True
+ }
+
+ # Add optional parameters if provided
+ if temperature is not None:
+ data["temperature"] = temperature
+ if top_p is not None:
+ data["top_p"] = top_p
+ if n is not None:
+ data["n"] = n
+ if max_tokens is not None:
+ data["max_tokens"] = max_tokens
+ if presence_penalty is not None:
+ data["presence_penalty"] = presence_penalty
+ if frequency_penalty is not None:
+ data["frequency_penalty"] = frequency_penalty
+ if user is not None:
+ data["user"] = user
+
+ # Make streaming request
+ session = requests.Session()
+ try:
+ response = session.post(
+ url,
+ headers=self._get_headers(),
+ json=data,
+ stream=True
+ )
+ response.raise_for_status()
+
+ # Parse SSE stream
+ for line in response.iter_lines():
+ if line:
+ line = line.decode('utf-8')
+ if line.startswith('data: '):
+ data_str = line[6:] # Remove 'data: ' prefix
+ if data_str.strip() == '[DONE]':
+ break
+ try:
+ chunk = json.loads(data_str)
+ yield chunk
+ except json.JSONDecodeError:
+ continue
+
+ except requests.exceptions.HTTPError as e:
+ if e.response.status_code == 401:
+ raise UnauthorizedError(e)
+ raise
diff --git a/litellm/proxy/client/cli/commands/auth.py b/litellm/proxy/client/cli/commands/auth.py
index 9be74268059..78cbb5ecf82 100644
--- a/litellm/proxy/client/cli/commands/auth.py
+++ b/litellm/proxy/client/cli/commands/auth.py
@@ -281,12 +281,12 @@ def prompt_team_selection_fallback(teams: List[Dict[str, Any]]) -> Optional[Dict
def update_key_with_team(base_url: str, api_key: str, team_id: str) -> bool:
"""Update the API key to be associated with the selected team"""
-
+ from litellm.proxy._types import SpecialModelNames
from litellm.proxy.client import Client
client = Client(base_url=base_url, api_key=api_key)
try:
- result = client.keys.update(key=api_key, team_id=team_id)
+ client.keys.update(key=api_key, team_id=team_id, models=[SpecialModelNames.all_team_models.value])
click.echo(f"✅ Successfully assigned key to team: {team_id}")
return True
except requests.exceptions.HTTPError as e:
@@ -301,6 +301,37 @@ def update_key_with_team(base_url: str, api_key: str, team_id: str) -> bool:
# Polling-based authentication - no local server needed
+def _handle_team_assignment(base_url: str, api_key: str, user_id: str) -> None:
+ """Handle team fetching and assignment for the authenticated user."""
+ click.echo("\n" + "="*60)
+ click.echo("📋 Fetching your teams...")
+
+ teams = get_user_teams(
+ base_url=base_url,
+ api_key=api_key,
+ user_id=user_id,
+ )
+
+ if teams:
+ # Prompt for team selection (will display teams interactively)
+ selected_team = prompt_team_selection(teams)
+
+ if selected_team:
+ team_id = selected_team.get('team_id')
+ if team_id:
+ click.echo(f"\n🔄 Assigning your key to team: {selected_team.get('team_alias', team_id)}")
+ success = update_key_with_team(base_url, api_key, team_id)
+ if success:
+ click.echo(f"✅ Your CLI key is now associated with team: {selected_team.get('team_alias', team_id)}")
+ click.echo(f"🎯 You can now access models: {', '.join(selected_team.get('models', ['All models']))}")
+ else:
+ click.echo("⚠️ Key assignment failed, but you can still use the CLI")
+ else:
+ click.echo("ℹ️ Continuing without team assignment. You can assign a team later using the CLI.")
+ else:
+ click.echo("ℹ️ No teams found. You can create or join teams using the web interface.")
+
+
@click.command(name="login")
@click.pass_context
def login(ctx: click.Context):
@@ -364,35 +395,8 @@ def login(ctx: click.Context):
click.echo(f"API Key: {api_key[:20]}...")
click.echo("You can now use the CLI without specifying --api-key")
- # Fetch and display user's teams
- click.echo("\n" + "="*60)
- click.echo("📋 Fetching your teams...")
-
- teams = get_user_teams(
- base_url=base_url,
- api_key=api_key,
- user_id=data.get("user_id"),
- )
-
-
- if teams:
- # Prompt for team selection (will display teams interactively)
- selected_team = prompt_team_selection(teams)
-
- if selected_team:
- team_id = selected_team.get('team_id')
- if team_id:
- click.echo(f"\n🔄 Assigning your key to team: {selected_team.get('team_alias', team_id)}")
- success = update_key_with_team(base_url, api_key, team_id)
- if success:
- click.echo(f"✅ Your CLI key is now associated with team: {selected_team.get('team_alias', team_id)}")
- click.echo(f"🎯 You can now access models: {', '.join(selected_team.get('models', ['All models']))}")
- else:
- click.echo("⚠️ Key assignment failed, but you can still use the CLI")
- else:
- click.echo("ℹ️ Continuing without team assignment. You can assign a team later using the CLI.")
- else:
- click.echo("ℹ️ No teams found. You can create or join teams using the web interface.")
+ # Handle team assignment
+ _handle_team_assignment(base_url, api_key, data.get("user_id"))
# Show available commands after successful login
click.echo("\n" + "="*60)
diff --git a/litellm/proxy/client/cli/commands/chat.py b/litellm/proxy/client/cli/commands/chat.py
index ea51f74652a..41ded68ed08 100644
--- a/litellm/proxy/client/cli/commands/chat.py
+++ b/litellm/proxy/client/cli/commands/chat.py
@@ -1,42 +1,96 @@
import json
-from typing import Optional
+import sys
+from typing import Any, Dict, List, Optional
import click
-import rich
import requests
+from rich.console import Console
+from rich.panel import Panel
+from rich.prompt import Prompt
+from rich.table import Table
+from ... import Client
from ...chat import ChatClient
-@click.group()
-def chat():
- """Chat with models through the LiteLLM proxy server"""
- pass
+def _get_available_models(ctx: click.Context) -> List[Dict[str, Any]]:
+ """Get list of available models from the proxy server"""
+ try:
+ client = Client(base_url=ctx.obj["base_url"], api_key=ctx.obj["api_key"])
+ models_list = client.models.list()
+ # Ensure we return a list of dictionaries
+ if isinstance(models_list, list):
+ # Filter to ensure all items are dictionaries
+ return [model for model in models_list if isinstance(model, dict)]
+ return []
+ except Exception as e:
+ click.echo(f"Warning: Could not fetch models list: {e}", err=True)
+ return []
-@chat.command()
-@click.argument("model")
-@click.option(
- "--message",
- "-m",
- multiple=True,
- help="Messages in 'role:content' format (e.g. 'user:Hello'). Can be specified multiple times.",
-)
+def _select_model(console: Console, available_models: List[Dict[str, Any]]) -> Optional[str]:
+ """Interactive model selection"""
+ if not available_models:
+ console.print("[yellow]No models available or could not fetch models list.[/yellow]")
+ model_name = Prompt.ask("Please enter a model name")
+ return model_name if model_name.strip() else None
+
+ # Display available models in a table
+ table = Table(title="Available Models")
+ table.add_column("Index", style="cyan", no_wrap=True)
+ table.add_column("Model ID", style="green")
+ table.add_column("Owned By", style="yellow")
+ MAX_MODELS_TO_DISPLAY = 200
+
+ models_to_display: List[Dict[str, Any]] = available_models[:MAX_MODELS_TO_DISPLAY]
+ for i, model in enumerate(models_to_display): # Limit to first 200 models
+ table.add_row(
+ str(i + 1),
+ str(model.get("id", "")),
+ str(model.get("owned_by", ""))
+ )
+
+ if len(available_models) > MAX_MODELS_TO_DISPLAY:
+ console.print(f"\n[dim]... and {len(available_models) - MAX_MODELS_TO_DISPLAY} more models[/dim]")
+
+ console.print(table)
+
+ while True:
+ try:
+ choice = Prompt.ask(
+ "\nSelect a model by entering the index number (or type a model name directly)",
+ default="1"
+ ).strip()
+
+ # Try to parse as index
+ try:
+ index = int(choice) - 1
+ if 0 <= index < len(available_models):
+ return available_models[index]["id"]
+ else:
+ console.print(f"[red]Invalid index. Please enter a number between 1 and {len(available_models)}[/red]")
+ continue
+ except ValueError:
+ # Not a number, treat as model name
+ if choice:
+ return choice
+ else:
+ console.print("[red]Please enter a valid model name or index[/red]")
+ continue
+
+ except KeyboardInterrupt:
+ console.print("\n[yellow]Model selection cancelled.[/yellow]")
+ return None
+
+
+@click.command()
+@click.argument("model", required=False)
@click.option(
"--temperature",
"-t",
type=float,
- help="Sampling temperature between 0 and 2",
-)
-@click.option(
- "--top-p",
- type=float,
- help="Nucleus sampling parameter between 0 and 1",
-)
-@click.option(
- "--n",
- type=int,
- help="Number of completions to generate",
+ default=0.7,
+ help="Sampling temperature between 0 and 2 (default: 0.7)",
)
@click.option(
"--max-tokens",
@@ -44,65 +98,271 @@ def chat():
help="Maximum number of tokens to generate",
)
@click.option(
- "--presence-penalty",
- type=float,
- help="Presence penalty between -2.0 and 2.0",
-)
-@click.option(
- "--frequency-penalty",
- type=float,
- help="Frequency penalty between -2.0 and 2.0",
-)
-@click.option(
- "--user",
+ "--system",
+ "-s",
type=str,
- help="Unique identifier for the end user",
+ help="System message to set the behavior of the assistant",
)
@click.pass_context
-def completions(
+def chat(
ctx: click.Context,
- model: str,
- message: tuple[str, ...],
- temperature: Optional[float] = None,
- top_p: Optional[float] = None,
- n: Optional[int] = None,
+ model: Optional[str],
+ temperature: float,
max_tokens: Optional[int] = None,
- presence_penalty: Optional[float] = None,
- frequency_penalty: Optional[float] = None,
- user: Optional[str] = None,
+ system: Optional[str] = None,
):
- """Create a chat completion"""
- if not message:
- raise click.UsageError("At least one message is required")
-
- # Parse messages from role:content format
- messages = []
- for msg in message:
- try:
- role, content = msg.split(":", 1)
- messages.append({"role": role.strip(), "content": content.strip()})
- except ValueError:
- raise click.BadParameter(f"Invalid message format: {msg}. Expected format: 'role:content'")
-
+ """Interactive chat with streaming responses
+
+ Examples:
+
+ # Chat with a specific model
+ litellm-proxy chat gpt-4
+
+ # Chat without specifying model (will show model selection)
+ litellm-proxy chat
+
+ # Chat with custom settings
+ litellm-proxy chat gpt-4 --temperature 0.9 --system "You are a helpful coding assistant"
+ """
+ console = Console()
+
+ # If no model specified, show model selection
+ if not model:
+ available_models = _get_available_models(ctx)
+ model = _select_model(console, available_models)
+ if not model:
+ console.print("[red]No model selected. Exiting.[/red]")
+ return
+
client = ChatClient(ctx.obj["base_url"], ctx.obj["api_key"])
+
+ # Initialize conversation history
+ messages: List[Dict[str, Any]] = []
+
+ # Add system message if provided
+ if system:
+ messages.append({"role": "system", "content": system})
+
+ # Display welcome message
+ console.print(Panel.fit(
+ f"[bold blue]LiteLLM Interactive Chat[/bold blue]\n"
+ f"Model: [green]{model}[/green]\n"
+ f"Temperature: [yellow]{temperature}[/yellow]\n"
+ f"Max Tokens: [yellow]{max_tokens or 'unlimited'}[/yellow]\n\n"
+ f"Type your messages and press Enter. Type '/quit' or '/exit' to end the session.\n"
+ f"Type '/help' for more commands.",
+ title="🤖 Chat Session"
+ ))
+
try:
- response = client.completions(
+ while True:
+ # Get user input
+ try:
+ user_input = console.input("\n[bold cyan]You:[/bold cyan] ").strip()
+ except (EOFError, KeyboardInterrupt):
+ console.print("\n[yellow]Chat session ended.[/yellow]")
+ break
+
+ # Handle special commands
+ should_exit, messages, new_model = _handle_special_commands(
+ console, user_input, messages, system, ctx
+ )
+
+ if should_exit:
+ break
+ if new_model:
+ model = new_model
+
+ # Check if this was a special command that was handled (not a normal message)
+ if user_input.lower().startswith(('/quit', '/exit', '/q', '/help', '/clear', '/history', '/save', '/load', '/model')) or not user_input:
+ continue
+
+ # Add user message to conversation
+ messages.append({"role": "user", "content": user_input})
+
+ # Display assistant label
+ console.print("\n[bold green]Assistant:[/bold green]")
+
+ # Stream the response
+ assistant_content = _stream_response(
+ console=console,
+ client=client,
+ model=model,
+ messages=messages,
+ temperature=temperature,
+ max_tokens=max_tokens,
+ )
+
+ # Add assistant message to conversation history
+ if assistant_content:
+ messages.append({"role": "assistant", "content": assistant_content})
+ else:
+ console.print("[red]Error: No content received from the model[/red]")
+
+ except KeyboardInterrupt:
+ console.print("\n[yellow]Chat session interrupted.[/yellow]")
+
+
+def _show_help(console: Console):
+ """Show help for interactive chat commands"""
+ help_text = """
+[bold]Interactive Chat Commands:[/bold]
+
+[cyan]/help[/cyan] - Show this help message
+[cyan]/quit[/cyan] - Exit the chat session (also /exit, /q)
+[cyan]/clear[/cyan] - Clear conversation history
+[cyan]/history[/cyan] - Show conversation history
+[cyan]/model[/cyan] - Switch to a different model
+[cyan]/save [/cyan] - Save conversation to file
+[cyan]/load [/cyan] - Load conversation from file
+
+[bold]Tips:[/bold]
+- Your conversation history is maintained during the session
+- Use Ctrl+C to interrupt at any time
+- Responses are streamed in real-time
+- You can switch models mid-conversation with /model
+ """
+ console.print(Panel(help_text, title="Help"))
+
+
+def _show_history(console: Console, messages: List[Dict[str, Any]]):
+ """Show conversation history"""
+ if not messages:
+ console.print("[yellow]No conversation history.[/yellow]")
+ return
+
+ console.print(Panel.fit("[bold]Conversation History[/bold]", title="History"))
+
+ for i, message in enumerate(messages, 1):
+ role = message["role"]
+ content = message["content"]
+
+ if role == "system":
+ console.print(f"[dim]{i}. [bold magenta]System:[/bold magenta] {content}[/dim]")
+ elif role == "user":
+ console.print(f"{i}. [bold cyan]You:[/bold cyan] {content}")
+ elif role == "assistant":
+ console.print(f"{i}. [bold green]Assistant:[/bold green] {content[:100]}{'...' if len(content) > 100 else ''}")
+
+
+def _save_conversation(console: Console, messages: List[Dict[str, Any]], command: str):
+ """Save conversation to a file"""
+ parts = command.split()
+ if len(parts) < 2:
+ console.print("[red]Usage: /save [/red]")
+ return
+
+ filename = parts[1]
+ if not filename.endswith('.json'):
+ filename += '.json'
+
+ try:
+ with open(filename, 'w') as f:
+ json.dump(messages, f, indent=2)
+ console.print(f"[green]Conversation saved to {filename}[/green]")
+ except Exception as e:
+ console.print(f"[red]Error saving conversation: {e}[/red]")
+
+
+def _load_conversation(console: Console, command: str, system: Optional[str]) -> List[Dict[str, Any]]:
+ """Load conversation from a file"""
+ parts = command.split()
+ if len(parts) < 2:
+ console.print("[red]Usage: /load [/red]")
+ return []
+
+ filename = parts[1]
+ if not filename.endswith('.json'):
+ filename += '.json'
+
+ try:
+ with open(filename, 'r') as f:
+ messages = json.load(f)
+ console.print(f"[green]Conversation loaded from {filename}[/green]")
+ return messages
+ except FileNotFoundError:
+ console.print(f"[red]File not found: {filename}[/red]")
+ except Exception as e:
+ console.print(f"[red]Error loading conversation: {e}[/red]")
+
+ # Return empty list or just system message if load failed
+ if system:
+ return [{"role": "system", "content": system}]
+ return []
+
+
+def _handle_special_commands(
+ console: Console,
+ user_input: str,
+ messages: List[Dict[str, Any]],
+ system: Optional[str],
+ ctx: click.Context
+) -> tuple[bool, List[Dict[str, Any]], Optional[str]]:
+ """Handle special chat commands. Returns (should_exit, updated_messages, updated_model)"""
+ if user_input.lower() in ['/quit', '/exit', '/q']:
+ console.print("[yellow]Chat session ended.[/yellow]")
+ return True, messages, None
+ elif user_input.lower() == '/help':
+ _show_help(console)
+ return False, messages, None
+ elif user_input.lower() == '/clear':
+ new_messages = []
+ if system:
+ new_messages.append({"role": "system", "content": system})
+ console.print("[green]Conversation history cleared.[/green]")
+ return False, new_messages, None
+ elif user_input.lower() == '/history':
+ _show_history(console, messages)
+ return False, messages, None
+ elif user_input.lower().startswith('/save'):
+ _save_conversation(console, messages, user_input)
+ return False, messages, None
+ elif user_input.lower().startswith('/load'):
+ new_messages = _load_conversation(console, user_input, system)
+ return False, new_messages, None
+ elif user_input.lower() == '/model':
+ available_models = _get_available_models(ctx)
+ new_model = _select_model(console, available_models)
+ if new_model:
+ console.print(f"[green]Switched to model: {new_model}[/green]")
+ return False, messages, new_model
+ return False, messages, None
+ elif not user_input:
+ return False, messages, None
+
+ # Not a special command
+ return False, messages, None
+
+
+def _stream_response(console: Console, client: ChatClient, model: str, messages: List[Dict[str, Any]], temperature: float, max_tokens: Optional[int]) -> Optional[str]:
+ """Stream the model response and return the complete content"""
+ try:
+ assistant_content = ""
+ for chunk in client.completions_stream(
model=model,
messages=messages,
temperature=temperature,
- top_p=top_p,
- n=n,
max_tokens=max_tokens,
- presence_penalty=presence_penalty,
- frequency_penalty=frequency_penalty,
- user=user,
- )
- rich.print_json(data=response)
+ ):
+ if "choices" in chunk and len(chunk["choices"]) > 0:
+ delta = chunk["choices"][0].get("delta", {})
+ content = delta.get("content", "")
+ if content:
+ assistant_content += content
+ console.print(content, end="")
+ sys.stdout.flush()
+
+ console.print() # Add newline after streaming
+ return assistant_content if assistant_content else None
+
except requests.exceptions.HTTPError as e:
- click.echo(f"Error: HTTP {e.response.status_code}", err=True)
+ console.print(f"\n[red]Error: HTTP {e.response.status_code}[/red]")
try:
error_body = e.response.json()
- rich.print_json(data=error_body)
+ console.print(f"[red]{error_body.get('error', {}).get('message', 'Unknown error')}[/red]")
except json.JSONDecodeError:
- click.echo(e.response.text, err=True)
- raise click.Abort()
+ console.print(f"[red]{e.response.text}[/red]")
+ return None
+ except Exception as e:
+ console.print(f"\n[red]Error: {str(e)}[/red]")
+ return None
\ No newline at end of file
diff --git a/litellm/proxy/client/cli/commands/teams.py b/litellm/proxy/client/cli/commands/teams.py
index d4e05c890dc..57397ca01a0 100644
--- a/litellm/proxy/client/cli/commands/teams.py
+++ b/litellm/proxy/client/cli/commands/teams.py
@@ -80,11 +80,8 @@ def list(ctx: click.Context):
display_teams_table(teams)
except requests.exceptions.HTTPError as e:
click.echo(f"Error: HTTP {e.response.status_code}", err=True)
- try:
- error_body = e.response.json()
- click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True)
- except:
- click.echo(e.response.text, err=True)
+ error_body = e.response.json()
+ click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True)
raise click.Abort()
except Exception as e:
click.echo(f"Error: {str(e)}", err=True)
@@ -107,12 +104,8 @@ def available(ctx: click.Context):
click.echo("ℹ️ No available teams to join.")
except requests.exceptions.HTTPError as e:
click.echo(f"Error: HTTP {e.response.status_code}", err=True)
- try:
- error_body = e.response.json()
- click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True)
- except:
- click.echo(e.response.text, err=True)
- raise click.Abort()
+ error_body = e.response.json()
+ click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True)
except Exception as e:
click.echo(f"Error: {str(e)}", err=True)
raise click.Abort()
@@ -152,7 +145,7 @@ def assign_key(ctx: click.Context, team_id: Optional[str]):
# Update the key with the selected team
if team_id:
click.echo(f"\n🔄 Assigning your key to team: {team_id}")
- result = client.keys.update(key=api_key, team_id=team_id)
+ client.keys.update(key=api_key, team_id=team_id)
click.echo(f"✅ Successfully assigned key to team: {team_id}")
# Show team details if available
@@ -168,11 +161,8 @@ def assign_key(ctx: click.Context, team_id: Optional[str]):
except requests.exceptions.HTTPError as e:
click.echo(f"Error: HTTP {e.response.status_code}", err=True)
- try:
- error_body = e.response.json()
- click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True)
- except:
- click.echo(e.response.text, err=True)
+ error_body = e.response.json()
+ click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True)
raise click.Abort()
except Exception as e:
click.echo(f"Error: {str(e)}", err=True)
diff --git a/litellm/proxy/client/cli/interface.py b/litellm/proxy/client/cli/interface.py
index 78aeb442351..2c6f5f10b4f 100644
--- a/litellm/proxy/client/cli/interface.py
+++ b/litellm/proxy/client/cli/interface.py
@@ -84,7 +84,7 @@ def show_commands():
("whoami", "Show current authentication status"),
("models", "Manage and view model configurations"),
("credentials", "Manage API credentials"),
- ("chat", "Interactive chat with models"),
+ ("chat", "Interactive streaming chat with models"),
("http", "Make HTTP requests to the proxy"),
("keys", "Manage API keys"),
("teams", "Manage teams and team assignments"),
diff --git a/litellm/proxy/client/client.py b/litellm/proxy/client/client.py
index 8ed9a4ff89f..c9066f70de6 100644
--- a/litellm/proxy/client/client.py
+++ b/litellm/proxy/client/client.py
@@ -1,5 +1,7 @@
from typing import Optional
+from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
+
from .chat import ChatClient
from .credentials import CredentialsManagementClient
from .http_client import HTTPClient
@@ -27,7 +29,7 @@ class Client:
timeout: Request timeout in seconds (default: 30)
"""
self._base_url = base_url.rstrip("/") # Remove trailing slash if present
- self._api_key = api_key
+ self._api_key = get_litellm_gateway_api_key() or api_key
# Initialize resource clients
diff --git a/litellm/proxy/client/keys.py b/litellm/proxy/client/keys.py
index fc307648a2f..50fd7b9d9c9 100644
--- a/litellm/proxy/client/keys.py
+++ b/litellm/proxy/client/keys.py
@@ -274,14 +274,15 @@ class KeysManagementClient:
data["aliases"] = aliases
request = requests.Request("POST", url, headers=self._get_headers(), json=data)
session = requests.Session()
+ response_text: Optional[str] = None
try:
response = session.send(request.prepare())
+ response_text = response.text
response.raise_for_status()
return response.json()
- except requests.exceptions.HTTPError as e:
- if e.response.status_code == 401:
- raise UnauthorizedError(e)
- raise
+ except Exception:
+ raise Exception(f"Error updating key: {response_text}")
+
def info(self, key: str, return_request: bool = False) -> Union[Dict[str, Any], requests.Request]:
"""
diff --git a/litellm/proxy/client/teams.py b/litellm/proxy/client/teams.py
index 61ddbe6adae..4f54b6bbd07 100644
--- a/litellm/proxy/client/teams.py
+++ b/litellm/proxy/client/teams.py
@@ -1,6 +1,6 @@
"""Teams management client for LiteLLM proxy."""
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
import requests
@@ -99,7 +99,7 @@ class TeamsManagementClient:
UnauthorizedError: If authentication fails
"""
url = f"{self._base_url}/v2/team/list"
- params = {
+ params: Dict[str, Union[str, int]] = {
"page": page,
"page_size": page_size,
"sort_order": sort_order,
diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index 5739e652043..95c84b914b6 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -14,7 +14,6 @@ from typing import (
Union,
)
-import fastuuid as uuid
import httpx
import orjson
from fastapi import HTTPException, Request, status
@@ -22,6 +21,7 @@ from fastapi.responses import Response, StreamingResponse
import litellm
from litellm._logging import verbose_proxy_logger
+from litellm._uuid import uuid
from litellm.constants import (
DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE,
STREAM_SSE_DATA_PREFIX,
diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py
index d52592952bc..fb7ada8ab10 100644
--- a/litellm/proxy/common_utils/callback_utils.py
+++ b/litellm/proxy/common_utils/callback_utils.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Literal, Optional
import litellm
from litellm import get_secret
@@ -289,7 +289,7 @@ def initialize_callbacks_on_proxy( # noqa: PLR0915
def get_model_group_from_litellm_kwargs(kwargs: dict) -> Optional[str]:
_litellm_params = kwargs.get("litellm_params", None) or {}
- _metadata = _litellm_params.get("metadata", None) or {}
+ _metadata = _litellm_params.get(get_metadata_variable_name_from_kwargs(kwargs)) or {}
_model_group = _metadata.get("model_group", None)
if _model_group is not None:
return _model_group
@@ -365,3 +365,20 @@ def add_guardrail_to_applied_guardrails_header(
_metadata["applied_guardrails"].append(guardrail_name)
else:
_metadata["applied_guardrails"] = [guardrail_name]
+
+
+def get_metadata_variable_name_from_kwargs(
+ kwargs: dict
+ ) -> Literal["metadata", "litellm_metadata"]:
+ """
+ Helper to return what the "metadata" field should be called in the request data
+
+ - New endpoints return `litellm_metadata`
+ - Old endpoints return `metadata`
+
+ Context:
+ - LiteLLM used `metadata` as an internal field for storing metadata
+ - OpenAI then started using this field for their metadata
+ - LiteLLM is now moving to using `litellm_metadata` for our metadata
+ """
+ return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py
index 480be4a651b..787c46d0dda 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py
@@ -21,11 +21,11 @@ from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
)
-from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
+from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import (
@@ -88,11 +88,11 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
def _create_sanitize_request(
self, content: str, source: Literal["user_prompt", "model_response"]
) -> dict:
- """Create request body for Model Armor API."""
+ """Create request body for Model Armor API with correct camelCase field names."""
if source == "user_prompt":
- return {"user_prompt_data": {"text": content}}
+ return {"userPromptData": {"text": content}}
else:
- return {"model_response_data": {"text": content}}
+ return {"modelResponseData": {"text": content}}
def _extract_content_from_response(
self, response: Union[Any, ModelResponse]
@@ -119,11 +119,16 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
async def make_model_armor_request(
self,
- content: str,
- source: Literal["user_prompt", "model_response"],
+ content: Optional[str] = None,
+ source: Literal["user_prompt", "model_response"] = "user_prompt",
request_data: Optional[dict] = None,
+ file_bytes: Optional[bytes] = None,
+ file_type: Optional[str] = None,
) -> dict:
- """Make request to Model Armor API."""
+ """
+ Make request to Model Armor API. Supports both text and file prompt sanitization.
+ If file_bytes and file_type are provided, file prompt sanitization is performed.
+ """
# Get access token using VertexBase auth
access_token, resolved_project_id = await self._ensure_access_token_async(
credentials=self.credentials,
@@ -143,7 +148,14 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
url = f"{endpoint}/v1/projects/{self.project_id}/locations/{self.location}/templates/{self.template_id}:sanitizeModelResponse"
# Create request body
- body = self._create_sanitize_request(content, source)
+ if file_bytes is not None and file_type is not None:
+ body = self.sanitize_file_prompt(file_bytes, file_type, source)
+ elif content is not None:
+ body = self._create_sanitize_request(content, source)
+ else:
+ raise ValueError(
+ "Either content or file_bytes and file_type must be provided."
+ )
# Set headers
headers = {
@@ -189,57 +201,112 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
return await json_response
return json_response
- def _should_block_content(self, armor_response: dict) -> bool:
- """Check if Model Armor response indicates content should be blocked."""
- # Check the sanitizationResult from Model Armor API
+ def sanitize_file_prompt(
+ self, file_bytes: bytes, file_type: str, source: str = "user_prompt"
+ ) -> dict:
+ """
+ Helper to build the request body for file prompt sanitization for Model Armor.
+ file_type should be one of: PLAINTEXT_UTF8, PDF, WORD_DOCUMENT, EXCEL_DOCUMENT, POWERPOINT_DOCUMENT, TXT, CSV
+ Returns the request body dict.
+ """
+ import base64
+
+ base64_data = base64.b64encode(file_bytes).decode("utf-8")
+ if source == "user_prompt":
+ return {
+ "userPromptData": {
+ "byteItem": {"byteDataType": file_type, "byteData": base64_data}
+ }
+ }
+ else:
+ return {
+ "modelResponseData": {
+ "byteItem": {"byteDataType": file_type, "byteData": base64_data}
+ }
+ }
+
+ def _should_block_content(self, armor_response: dict, allow_sanitization: bool = False) -> bool:
+ """Check if Model Armor response indicates content should be blocked, including both inspectResult and deidentifyResult."""
sanitization_result = armor_response.get("sanitizationResult", {})
filter_results = sanitization_result.get("filterResults", {})
- # Check blocking filters (these should cause the request to be blocked)
- # RAI (Responsible AI) filters
- rai_results = filter_results.get("rai", {}).get("raiFilterResult", {})
- if rai_results.get("matchState") == "MATCH_FOUND":
- return True
-
- # Prompt injection and jailbreak filters
- pi_jailbreak = filter_results.get("piAndJailbreakFilterResult", {})
- if pi_jailbreak.get("matchState") == "MATCH_FOUND":
- return True
-
- # Malicious URI filters
- malicious_uri = filter_results.get("maliciousUriFilterResult", {})
- if malicious_uri.get("matchState") == "MATCH_FOUND":
- return True
-
- # CSAM filters
- csam = filter_results.get("csamFilterFilterResult", {})
- if csam.get("matchState") == "MATCH_FOUND":
- return True
-
- # Virus scan filters
- virus_scan = filter_results.get("virusScanFilterResult", {})
- if virus_scan.get("matchState") == "MATCH_FOUND":
- return True
+ # filterResults can be a dict (named keys) or a list (array of filter result dicts)
+ filter_result_items = []
+ if isinstance(filter_results, dict):
+ filter_result_items = list(filter_results.values())
+ elif isinstance(filter_results, list):
+ filter_result_items = filter_results
+ for filt in filter_result_items:
+ # Check RAI, PI/Jailbreak, Malicious URI, CSAM, Virus scan as before
+ if filt.get("raiFilterResult", {}).get("matchState") == "MATCH_FOUND":
+ return True
+ if (
+ filt.get("piAndJailbreakFilterResult", {}).get("matchState")
+ == "MATCH_FOUND"
+ ):
+ return True
+ if (
+ filt.get("maliciousUriFilterResult", {}).get("matchState")
+ == "MATCH_FOUND"
+ ):
+ return True
+ if (
+ filt.get("csamFilterFilterResult", {}).get("matchState")
+ == "MATCH_FOUND"
+ ):
+ return True
+ if filt.get("virusScanFilterResult", {}).get("matchState") == "MATCH_FOUND":
+ return True
+ # Check sdpFilterResult for both inspectResult and deidentifyResult
+ sdp = filt.get("sdpFilterResult")
+ if sdp:
+ if sdp.get("inspectResult", {}).get("matchState") == "MATCH_FOUND":
+ return True
+ # Only block on deidentifyResult if sanitization is not allowed
+ if sdp.get("deidentifyResult", {}).get("matchState") == "MATCH_FOUND":
+ if not allow_sanitization:
+ return True
+ # Fallback dict code removed; all cases handled above
return False
def _get_sanitized_content(self, armor_response: dict) -> Optional[str]:
- """Extract sanitized content from Model Armor response."""
- # Model Armor returns sanitized content in the sanitizationResult
- sanitization_result = armor_response.get("sanitizationResult", {})
+ """
+ Get the sanitized content from a Model Armor response, if available.
+ Looks for sanitized text in deidentifyResult, and falls back to root-level fields if not found.
+ """
+ result = armor_response.get("sanitizationResult", {})
+ filter_results = result.get("filterResults", {})
- # Check for sdp structure (for deidentification)
- filter_results = sanitization_result.get("filterResults", {})
- sdp = filter_results.get("sdp", {}).get("sdpFilterResult")
+ # filterResults can be a dict (single filter) or a list (multiple filters)
+ filters = (
+ list(filter_results.values())
+ if isinstance(filter_results, dict)
+ else filter_results
+ if isinstance(filter_results, list)
+ else []
+ )
- if sdp is not None:
- # Model Armor returns sanitized text under deidentifyResult in sdp
- deidentify_result = sdp.get("deidentifyResult", {})
- sanitized_text = deidentify_result.get("data", {}).get("text", "")
- if deidentify_result.get("matchState") == "MATCH_FOUND" and sanitized_text:
- return sanitized_text
+ # Prefer sanitized text from deidentifyResult if present
+ for filter_entry in filters:
+ sdp = filter_entry.get("sdpFilterResult")
+ if sdp:
+ deid = sdp.get("deidentifyResult", {})
+ sanitized = deid.get("data", {}).get("text", "")
+ # If Model Armor found something and returned a sanitized version, use it
+ if deid.get("matchState") == "MATCH_FOUND" and sanitized:
+ return sanitized
- # Fallback to checking root level
+ # If no deidentifyResult, optionally check for inspectResult (rare, but could have findings)
+ for filter_entry in filters:
+ sdp = filter_entry.get("sdpFilterResult")
+ if sdp:
+ inspect = sdp.get("inspectResult", {})
+ # If Model Armor flagged something but didn't sanitize, return None
+ if inspect.get("matchState") == "MATCH_FOUND":
+ return None
+
+ # Fallback: if Model Armor put sanitized text at the root, use it
return armor_response.get("sanitizedText") or armor_response.get("text")
def _process_response(
@@ -344,11 +411,11 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
# fail_on_error=False) we still want the correct status reflected.
metadata["_model_armor_status"] = (
"blocked"
- if self._should_block_content(armor_response)
+ if self._should_block_content(armor_response, allow_sanitization=self.mask_request_content)
else "success"
)
# Check if content should be blocked
- if self._should_block_content(armor_response):
+ if self._should_block_content(armor_response, allow_sanitization=self.mask_request_content):
raise HTTPException(
status_code=400,
detail={
@@ -429,12 +496,12 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
metadata["_model_armor_response"] = armor_response
metadata["_model_armor_status"] = (
"blocked"
- if self._should_block_content(armor_response)
+ if self._should_block_content(armor_response, allow_sanitization=self.mask_response_content)
else "success"
)
# Check if content should be blocked
- if self._should_block_content(armor_response):
+ if self._should_block_content(armor_response, allow_sanitization=self.mask_response_content):
raise HTTPException(
status_code=400,
detail={
diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py
index fef03d54743..38e211dea50 100644
--- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py
+++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py
@@ -40,7 +40,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
def _get_priority_weight(self, priority: Optional[str]) -> float:
"""Get the weight for a given priority from litellm.priority_reservation"""
- weight: float = 1.0
+ weight: float = litellm.priority_reservation_settings.default_priority
if (
litellm.priority_reservation is None
or priority not in litellm.priority_reservation
diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py
index 8ef0a662ffe..b6b82e4b376 100644
--- a/litellm/proxy/hooks/parallel_request_limiter_v3.py
+++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py
@@ -25,6 +25,7 @@ from litellm import DualCache
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
+from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
@@ -708,6 +709,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
from litellm.proxy.common_utils.callback_utils import (
get_model_group_from_litellm_kwargs,
+ get_metadata_variable_name_from_kwargs
)
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.utils import ModelResponse, Usage
@@ -723,7 +725,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
# Get metadata from kwargs
- litellm_metadata = kwargs["litellm_params"]["metadata"]
+ litellm_metadata = kwargs["litellm_params"].get(get_metadata_variable_name_from_kwargs(kwargs), {})
if litellm_metadata is None:
return
user_api_key = litellm_metadata.get("user_api_key")
@@ -736,7 +738,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# Get total tokens from response
total_tokens = 0
- if isinstance(response_obj, ModelResponse):
+ # spot fix for /responses api
+ if (isinstance(response_obj, ModelResponse) or isinstance(response_obj, BaseLiteLLMOpenAIResponseObject)):
_usage = getattr(response_obj, "usage", None)
if _usage and isinstance(_usage, Usage):
if rate_limit_type == "output":
diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py
index 026ae9e4056..eb389450f91 100644
--- a/litellm/proxy/management_endpoints/key_management_endpoints.py
+++ b/litellm/proxy/management_endpoints/key_management_endpoints.py
@@ -1232,13 +1232,11 @@ def validate_key_team_change(
)
# Check if the key's user_id is a member of the team
+ member_object = _get_user_in_team(
+ team_table=cast(LiteLLM_TeamTableCachedObj, team), user_id=key.user_id
+ )
if key.user_id is not None:
- is_member = False
- for member in team.members_with_roles:
- if member.user_id == key.user_id:
- is_member = True
- break
- if not is_member:
+ if not member_object:
raise HTTPException(
status_code=403,
detail=f"User={key.user_id} is not a member of the team={team.team_id}. Check team members via `/team/info`.",
@@ -1265,10 +1263,17 @@ def validate_key_team_change(
team_obj=team,
):
return
+ # this teams member permissions allow updating a
+ elif TeamMemberPermissionChecks.does_team_member_have_permissions_for_endpoint(
+ team_member_object=member_object,
+ team_table=cast(LiteLLM_TeamTableCachedObj, team),
+ route=KeyManagementRoutes.KEY_UPDATE.value,
+ ):
+ return
else:
raise HTTPException(
status_code=403,
- detail=f"User={change_initiated_by.user_id} is not a Proxy Admin or Team Admin for team={team.team_id}.",
+ detail=f"User={change_initiated_by.user_id} is not a Proxy Admin or Team Admin for team={team.team_id}. Please ask your Proxy Admin to allow this action under 'Member Permissions' for this team.",
)
diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py
index 3828f7318c1..2af940ec654 100644
--- a/litellm/proxy/management_endpoints/ui_sso.py
+++ b/litellm/proxy/management_endpoints/ui_sso.py
@@ -731,7 +731,6 @@ async def _create_new_cli_key(
config={},
spend=0,
user_id=user_id,
- team_id="litellm-cli",
table_name="key",
token=key,
)
@@ -810,7 +809,7 @@ async def cli_poll_key(key_id: str):
key_obj = await prisma_client.db.litellm_verificationtoken.find_unique(
where={"token": hashed_token}
)
- key_obj: LiteLLM_VerificationToken = cast(LiteLLM_VerificationToken, key_obj)
+ key_obj = cast(LiteLLM_VerificationToken, key_obj)
if key_obj:
verbose_proxy_logger.info(f"CLI key found: {key_id}")
@@ -1503,7 +1502,7 @@ class SSOAuthenticationHandler:
## CHECK IF ROLE ALLOWED TO USE PROXY ##
is_admin_only_access = check_is_admin_only_access(ui_access_mode or {})
if is_admin_only_access:
- has_access = has_admin_ui_access(user_role)
+ has_access = has_admin_ui_access(user_role or "")
if not has_access:
raise HTTPException(
status_code=401,
@@ -1547,7 +1546,7 @@ class SSOAuthenticationHandler:
user_id=cast(str, user_id),
key=key,
user_email=user_email,
- user_role=user_role,
+ user_role=user_role or LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value,
login_method="sso",
premium_user=premium_user,
auth_header_name=general_settings.get(
diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
index 5e171af5252..a834a7a13c3 100644
--- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
@@ -459,6 +459,14 @@ async def anthropic_proxy_route(
region_name=None,
)
+ custom_headers = {}
+ if (
+ "authorization" not in request.headers
+ and "x-api-key" not in request.headers
+ and anthropic_api_key is not None
+ ):
+ custom_headers["x-api-key"] = "{}".format(anthropic_api_key)
+
## check for streaming
is_streaming_request = await is_streaming_request_fn(request)
@@ -466,7 +474,7 @@ async def anthropic_proxy_route(
endpoint_func = create_pass_through_route(
endpoint=endpoint,
target=str(updated_url),
- custom_headers={"x-api-key": "{}".format(anthropic_api_key)},
+ custom_headers=custom_headers,
_forward_headers=True,
) # dynamically construct pass-through endpoint based on incoming path
received_value = await endpoint_func(
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 52f42c9f310..472aeb140cf 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -35,6 +35,7 @@ from litellm.constants import (
LITELLM_SETTINGS_SAFE_DB_OVERRIDES,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.utils import load_credentials_from_list
from litellm.types.utils import (
ModelResponse,
ModelResponseStream,
@@ -5894,6 +5895,7 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False)
pass
if deployment is not None:
litellm_model_name = deployment.get("litellm_params", {}).get("model")
+ load_credentials_from_list(deployment.get("litellm_params", {}))
# remove the custom_llm_provider_prefix in the litellm_model_name
if "/" in litellm_model_name:
litellm_model_name = litellm_model_name.split("/", 1)[1]
diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
index 83acf0e8226..e2ca3078384 100644
--- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
+++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
@@ -493,21 +493,32 @@ async def update_sso_settings(sso_config: SSOConfig):
config["general_settings"] = {}
# Update environment variables in config and in memory
- sso_data = sso_config.model_dump(exclude_none=True)
+ sso_data = sso_config.model_dump()
for field_name, value in sso_data.items():
-
- if field_name == "user_email" and value is not None:
- # Store user_email in general_settings instead of environment variables
- config["general_settings"]["proxy_admin_email"] = value
- elif field_name == "ui_access_mode" and value is not None:
-
- config["general_settings"]["ui_access_mode"] = value
- elif field_name in env_var_mapping and value is not None:
+ if field_name == "user_email":
+ if value:
+ # Store user_email in general_settings instead of environment variables
+ config["general_settings"]["proxy_admin_email"] = value
+ else:
+ # Clear user_email if null/empty
+ config["general_settings"].pop("proxy_admin_email", None)
+ elif field_name == "ui_access_mode":
+ if value:
+ config["general_settings"]["ui_access_mode"] = value
+ else:
+ # Clear ui_access_mode if null/empty
+ config["general_settings"].pop("ui_access_mode", None)
+ elif field_name in env_var_mapping and value:
env_var_name = env_var_mapping[field_name]
# Update in config
config["environment_variables"][env_var_name] = value
# Update in runtime environment
os.environ[env_var_name] = value
+ elif field_name in env_var_mapping:
+ # Clear environment variable if value is null/empty
+ env_var_name = env_var_mapping[field_name]
+ config["environment_variables"].pop(env_var_name, None)
+ os.environ.pop(env_var_name, None)
stored_config = config
if len(config["environment_variables"]) > 0:
diff --git a/litellm/types/integrations/langfuse.py b/litellm/types/integrations/langfuse.py
index e509d074172..08ad667cac4 100644
--- a/litellm/types/integrations/langfuse.py
+++ b/litellm/types/integrations/langfuse.py
@@ -7,3 +7,10 @@ class LangfuseLoggingConfig(TypedDict):
langfuse_secret: Optional[str]
langfuse_public_key: Optional[str]
langfuse_host: Optional[str]
+
+
+class LangfuseUsageDetails(TypedDict):
+ input: Optional[int]
+ output: Optional[int]
+ cache_creation_input_tokens: Optional[int]
+ cache_read_input_tokens: Optional[int]
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index 01bf59fc841..d37978f3e21 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -13,7 +13,6 @@ from typing import (
Union,
)
-import fastuuid as uuid
from aiohttp import FormData
from openai._models import BaseModel as OpenAIObject
from openai.types.audio.transcription_create_params import FileTypes # type: ignore
@@ -33,6 +32,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
from typing_extensions import Callable, Dict, Required, TypedDict, override
import litellm
+from litellm._uuid import uuid
from litellm.types.llms.base import (
BaseLiteLLMOpenAIResponseObject,
LiteLLMPydanticObjectBase,
@@ -2633,3 +2633,18 @@ CostResponseTypes = Union[
ImageResponse,
TranscriptionResponse,
]
+
+
+class PriorityReservationSettings(BaseModel):
+ """
+ Settings for priority-based rate limiting reservation.
+
+ Defines what priority to assign to keys without explicit priority metadata.
+ The priority_reservation mapping is configured separately via litellm.priority_reservation.
+ """
+ default_priority: float = Field(
+ default=0.5,
+ description="Priority level to assign to API keys without explicit priority metadata. Should match a key in litellm.priority_reservation."
+ )
+
+ model_config = ConfigDict(protected_namespaces=())
diff --git a/litellm/utils.py b/litellm/utils.py
index 3c3ab0832d7..0721b023d2b 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -40,7 +40,6 @@ from os.path import abspath, dirname, join
import aiohttp
import dotenv
-import fastuuid as uuid
import httpx
import openai
import tiktoken
@@ -59,6 +58,7 @@ import litellm.litellm_core_utils.audio_utils.utils
import litellm.litellm_core_utils.json_validation_rule
import litellm.llms
import litellm.llms.gemini
+from litellm._uuid import uuid
from litellm.caching._internal_lru_cache import lru_cache_wrapper
from litellm.caching.caching import DualCache
from litellm.caching.caching_handler import CachingHandlerResponse, LLMCachingHandler
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index fa0f204887c..de1f0c5724e 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -560,10 +560,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -824,10 +828,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -4817,10 +4825,14 @@
"cache_creation_input_token_cost_above_1hr": 6e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "anthropic",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -7264,6 +7276,18 @@
"supports_prompt_caching": true,
"supports_tool_choice": true
},
+ "deepseek.v3-v1:0": {
+ "input_cost_per_token": 5.8e-07,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 163840,
+ "max_output_tokens": 81920,
+ "max_tokens": 163840,
+ "mode": "chat",
+ "output_cost_per_token": 1.68e-06,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
"dolphin": {
"input_cost_per_token": 5e-07,
"litellm_provider": "nlp_cloud",
@@ -7620,10 +7644,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -16260,10 +16288,12 @@
"openrouter/anthropic/claude-sonnet-4": {
"input_cost_per_image": 0.0048,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
"litellm_provider": "openrouter",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"supports_assistant_prefill": true,
@@ -17605,6 +17635,54 @@
"mode": "chat",
"output_cost_per_token": 2.8e-07
},
+ "qwen.qwen3-coder-480b-a35b-v1:0": {
+ "input_cost_per_token": 2.2e-07,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 262000,
+ "max_output_tokens": 65536,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 1.8e-06,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
+ "qwen.qwen3-235b-a22b-2507-v1:0": {
+ "input_cost_per_token": 2.2e-07,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 131072,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 8.8e-07,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
+ "qwen.qwen3-coder-30b-a3b-v1:0": {
+ "input_cost_per_token": 1.5e-07,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 131072,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 6.0e-07,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
+ "qwen.qwen3-32b-v1:0": {
+ "input_cost_per_token": 1.5e-07,
+ "litellm_provider": "bedrock_converse",
+ "max_input_tokens": 131072,
+ "max_output_tokens": 16384,
+ "max_tokens": 131072,
+ "mode": "chat",
+ "output_cost_per_token": 6.0e-07,
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true
+ },
"recraft/recraftv2": {
"litellm_provider": "recraft",
"mode": "image_generation",
@@ -19123,10 +19201,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "bedrock_converse",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -20459,10 +20541,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "vertex_ai-anthropic_models",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -20485,10 +20571,14 @@
"cache_creation_input_token_cost": 3.75e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
+ "input_cost_per_token_above_200k_tokens": 6e-06,
+ "output_cost_per_token_above_200k_tokens": 2.25e-05,
+ "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+ "cache_read_input_token_cost_above_200k_tokens": 6e-07,
"litellm_provider": "vertex_ai-anthropic_models",
- "max_input_tokens": 200000,
- "max_output_tokens": 64000,
- "max_tokens": 64000,
+ "max_input_tokens": 1000000,
+ "max_output_tokens": 1000000,
+ "max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"search_context_cost_per_query": {
@@ -21334,6 +21424,30 @@
"/v1/audio/transcriptions"
]
},
+ "vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": {
+ "input_cost_per_token": 1.5e-07,
+ "litellm_provider": "vertex_ai-qwen_models",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+ "supports_function_calling": true,
+ "supports_tool_choice": true
+ },
+ "vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": {
+ "input_cost_per_token": 1.5e-07,
+ "litellm_provider": "vertex_ai-qwen_models",
+ "max_input_tokens": 262144,
+ "max_output_tokens": 262144,
+ "max_tokens": 262144,
+ "mode": "chat",
+ "output_cost_per_token": 1.2e-06,
+ "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+ "supports_function_calling": true,
+ "supports_tool_choice": true
+ },
"xai/grok-2": {
"input_cost_per_token": 2e-06,
"litellm_provider": "xai",
@@ -21586,6 +21700,35 @@
"supports_tool_choice": true,
"supports_web_search": true
},
+ "xai/grok-4-fast-reasoning": {
+ "litellm_provider": "xai",
+ "max_input_tokens": 2e6,
+ "max_output_tokens": 2e6,
+ "max_tokens": 2e6,
+ "mode": "chat",
+ "input_cost_per_token": 0.2e-06,
+ "output_cost_per_token": 0.5e-06,
+ "cache_read_input_token_cost": 0.05e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_reasoning": true,
+ "supports_tool_choice": true,
+ "supports_web_search": true
+ },
+ "xai/grok-4-fast-non-reasoning": {
+ "litellm_provider": "xai",
+ "max_input_tokens": 2e6,
+ "max_output_tokens": 2e6,
+ "cache_read_input_token_cost": 0.05e-06,
+ "max_tokens": 2e6,
+ "mode": "chat",
+ "input_cost_per_token": 0.2e-06,
+ "output_cost_per_token": 0.5e-06,
+ "source": "https://docs.x.ai/docs/models",
+ "supports_function_calling": true,
+ "supports_tool_choice": true,
+ "supports_web_search": true
+ },
"xai/grok-4-0709": {
"input_cost_per_token": 3e-06,
"litellm_provider": "xai",
diff --git a/poetry.lock b/poetry.lock
index 4d0e3bea696..a8ef9ee79df 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -555,7 +555,7 @@ description = "Foreign Function Interface for Python calling C code."
optional = false
python-versions = ">=3.8"
groups = ["main", "dev", "proxy-dev"]
-markers = "python_version < \"3.14\" and platform_python_implementation != \"PyPy\""
+markers = "python_version < \"3.10\" and platform_python_implementation != \"PyPy\""
files = [
{file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"},
{file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"},
@@ -636,7 +636,7 @@ description = "Foreign Function Interface for Python calling C code."
optional = false
python-versions = ">=3.9"
groups = ["main", "dev", "proxy-dev"]
-markers = "platform_python_implementation != \"PyPy\" and python_version >= \"3.14\""
+markers = "python_version >= \"3.10\" and platform_python_implementation != \"PyPy\""
files = [
{file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"},
{file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"},
@@ -1339,9 +1339,10 @@ zstandard = ["zstandard"]
name = "fastuuid"
version = "0.12.0"
description = "Python bindings to Rust's UUID library."
-optional = false
+optional = true
python-versions = ">=3.8"
groups = ["main"]
+markers = "extra == \"proxy\""
files = [
{file = "fastuuid-0.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:22a900ef0956aacf862b460e20541fdae2d7c340594fe1bd6fdcb10d5f0791a9"},
{file = "fastuuid-0.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0302f5acf54dc75de30103025c5a95db06d6c2be36829043a0aa16fc170076bc"},
@@ -4361,7 +4362,7 @@ description = "C parser in Python"
optional = false
python-versions = ">=3.8"
groups = ["main", "dev", "proxy-dev"]
-markers = "platform_python_implementation != \"PyPy\" and (python_version < \"3.14\" or implementation_name != \"PyPy\")"
+markers = "platform_python_implementation != \"PyPy\" and (implementation_name != \"PyPy\" or python_version < \"3.10\")"
files = [
{file = "pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934"},
{file = "pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2"},
@@ -6746,11 +6747,11 @@ type = ["pytest-mypy"]
caching = ["diskcache"]
extra-proxy = ["azure-identity", "azure-keyvault-secrets", "google-cloud-iam", "google-cloud-kms", "prisma", "redisvl", "resend"]
mlflow = ["mlflow"]
-proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "python-multipart", "pyyaml", "rich", "rq", "uvicorn", "uvloop", "websockets"]
+proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "fastuuid", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "python-multipart", "pyyaml", "rich", "rq", "uvicorn", "uvloop", "websockets"]
semantic-router = ["semantic-router"]
utils = ["numpydoc"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.8.1,<4.0, !=3.9.7"
-content-hash = "dcec654bc4b233d2f0d160341d8adf1ba2017ccb74c104bff9ba4cf027ba0186"
+content-hash = "75004c6a23b70be86622fa417fd0d62fa3843e6e61c8dff8507ae5c967b7205d"
diff --git a/pyproject.toml b/pyproject.toml
index 81104d072ee..b11b1a1c2e2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
-version = "1.77.3"
+version = "1.77.4"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT"
@@ -20,7 +20,6 @@ Documentation = "https://docs.litellm.ai"
[tool.poetry.dependencies]
python = ">=3.8.1,<4.0, !=3.9.7"
-fastuuid = ">=0.12.0"
httpx = ">=0.23.0"
openai = ">=1.99.5"
python-dotenv = ">=0.2.0"
@@ -34,6 +33,7 @@ pydantic = "^2.5.0"
jsonschema = "^4.22.0"
pondpond = "^1.4.1"
numpydoc = {version = "*", optional = true} # used in utils.py
+fastuuid = {version = ">=0.12.0", optional = true}
uvicorn = {version = "^0.29.0", optional = true}
uvloop = {version = "^0.21.0", optional = true, markers="sys_platform != 'win32'"}
@@ -93,6 +93,7 @@ proxy = [
"litellm-enterprise",
"rich",
"polars",
+ "fastuuid",
]
extra_proxy = [
@@ -115,6 +116,7 @@ semantic-router = ["semantic-router"]
mlflow = ["mlflow"]
+
[tool.isort]
profile = "black"
@@ -157,7 +159,7 @@ requires = ["poetry-core", "wheel"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
-version = "1.77.3"
+version = "1.77.4"
version_files = [
"pyproject.toml:^version"
]
diff --git a/security.md b/security.md
index 2da073661c5..d126dabcc67 100644
--- a/security.md
+++ b/security.md
@@ -12,11 +12,6 @@
- For installation and configuration, see: [Self-hosting guided](https://docs.litellm.ai/docs/proxy/deploy)
- **Telemetry** We run no telemetry when you self host LiteLLM
-
-:::info
-✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
-:::
-
### LiteLLM Cloud
- We encrypt all data stored using your `LITELLM_MASTER_KEY` and in transit using TLS.
diff --git a/test_model_armor.py b/test_model_armor.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/code_coverage_tests/check_fastuuid_usage.py b/tests/code_coverage_tests/check_fastuuid_usage.py
new file mode 100644
index 00000000000..e0433454371
--- /dev/null
+++ b/tests/code_coverage_tests/check_fastuuid_usage.py
@@ -0,0 +1,87 @@
+import ast
+import os
+from typing import List, Dict, Any
+
+
+ALLOWED_FILE = os.path.normpath("litellm/_uuid.py")
+
+
+def _to_module_path(relative_path: str) -> str:
+ module = os.path.splitext(relative_path)[0].replace(os.sep, ".")
+ if module.endswith(".__init__"):
+ return module[: -len(".__init__")]
+ return module
+
+
+def _find_fastuuid_imports_in_file(
+ file_path: str, base_dir: str
+) -> List[Dict[str, Any]]:
+ results: List[Dict[str, Any]] = []
+ try:
+ with open(file_path, "r", encoding="utf-8") as f:
+ source = f.read()
+ tree = ast.parse(source, filename=file_path)
+ except Exception:
+ return results
+
+ relative = os.path.normpath(os.path.relpath(file_path, base_dir))
+ if relative == ALLOWED_FILE:
+ return results
+
+ module = _to_module_path(relative)
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Import):
+ for alias in node.names:
+ if alias.name == "fastuuid":
+ results.append(
+ {
+ "file": relative,
+ "line": getattr(node, "lineno", 0),
+ "import": f"import {alias.name}",
+ "module": module,
+ }
+ )
+ elif isinstance(node, ast.ImportFrom) and node.module == "fastuuid":
+ names = ", ".join([a.name for a in node.names])
+ results.append(
+ {
+ "file": relative,
+ "line": getattr(node, "lineno", 0),
+ "import": f"from fastuuid import {names}",
+ "module": module,
+ }
+ )
+
+ return results
+
+
+def scan_directory_for_fastuuid(base_dir: str) -> List[Dict[str, Any]]:
+ violations: List[Dict[str, Any]] = []
+ scan_root = os.path.join(base_dir, "litellm")
+ for root, _, files in os.walk(scan_root):
+ for filename in files:
+ if filename.endswith(".py"):
+ file_path = os.path.join(root, filename)
+ violations.extend(_find_fastuuid_imports_in_file(file_path, base_dir))
+ return violations
+
+
+def main() -> None:
+ base_dir = "." # tests run from repo root in CI
+ violations = scan_directory_for_fastuuid(base_dir)
+ if violations:
+ print(
+ "\n🚨 fastuuid must only be imported inside litellm/_uuid.py. Found violations:"
+ )
+ for v in violations:
+ print(f"* {v['module']} ({v['file']}:{v['line']}) -> {v['import']}")
+ print("\n")
+ raise Exception(
+ "Found fastuuid imports outside litellm/_uuid.py. Use litellm._uuid.uuid or litellm._uuid.uuid4 instead."
+ )
+ else:
+ print("✅ No invalid fastuuid imports found.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/guardrails_tests/test_model_armor_file_sanitization.py b/tests/guardrails_tests/test_model_armor_file_sanitization.py
deleted file mode 100644
index 4c9826ee14d..00000000000
--- a/tests/guardrails_tests/test_model_armor_file_sanitization.py
+++ /dev/null
@@ -1,75 +0,0 @@
-import sys
-import os
-import pytest
-from unittest.mock import AsyncMock
-from fastapi import HTTPException
-
-sys.path.insert(0, os.path.abspath("../.."))
-
-from litellm.proxy.guardrails.guardrail_hooks.model_armor.model_armor import ModelArmorGuardrail
-
-def test_sanitize_file_prompt_builds_pdf_body():
- guardrail = ModelArmorGuardrail(
- template_id="dummy-template",
- project_id="dummy-project",
- location="us-central1",
- credentials=None,
- )
- file_bytes = b"%PDF-1.4 some pdf content"
- file_type = "PDF"
- body = guardrail.sanitize_file_prompt(file_bytes, file_type, source="user_prompt")
- assert "userPromptData" in body
- assert body["userPromptData"]["byteItem"]["byteDataType"] == "PDF"
- import base64
- assert body["userPromptData"]["byteItem"]["byteData"] == base64.b64encode(file_bytes).decode("utf-8")
-
-@pytest.mark.asyncio
-async def test_make_model_armor_request_file_prompt():
- guardrail = ModelArmorGuardrail(
- template_id="dummy-template",
- project_id="dummy-project",
- location="us-central1",
- credentials=None,
- )
- file_bytes = b"My SSN is 123-45-6789."
- file_type = "PLAINTEXT_UTF8"
- armor_response = {
- "sanitizationResult": {
- "filterResults": [
- {
- "sdpFilterResult": {
- "inspectResult": {
- "executionState": "EXECUTION_SUCCESS",
- "matchState": "MATCH_FOUND",
- "findings": [
- {"infoType": "US_SOCIAL_SECURITY_NUMBER", "likelihood": "LIKELY"}
- ]
- },
- "deidentifyResult": {
- "executionState": "EXECUTION_SUCCESS",
- "matchState": "MATCH_FOUND",
- "data": {"text": "My SSN is [REDACTED]."}
- }
- }
- }
- ]
- }
- }
- class MockResponse:
- def __init__(self, status_code, text, json_data):
- self.status_code = status_code
- self.text = text
- self._json = json_data
- def json(self):
- return self._json
- class MockHandler:
- async def post(self, url, json, headers):
- return MockResponse(200, str(armor_response), armor_response)
- guardrail.async_handler = MockHandler()
- guardrail._ensure_access_token_async = AsyncMock(return_value=("dummy-token", "dummy-project"))
- result = await guardrail.make_model_armor_request(
- file_bytes=file_bytes,
- file_type=file_type,
- source="user_prompt"
- )
- assert result["sanitizationResult"]["filterResults"][0]["sdpFilterResult"]["deidentifyResult"]["data"]["text"] == "My SSN is [REDACTED]."
diff --git a/tests/guardrails_tests/test_model_armor_guardrail.py b/tests/guardrails_tests/test_model_armor_guardrail.py
deleted file mode 100644
index 51e5859d081..00000000000
--- a/tests/guardrails_tests/test_model_armor_guardrail.py
+++ /dev/null
@@ -1,99 +0,0 @@
-import sys
-import os
-import pytest
-from unittest.mock import AsyncMock, patch
-from fastapi import HTTPException
-
-sys.path.insert(0, os.path.abspath("../.."))
-
-from litellm.proxy.guardrails.guardrail_hooks.model_armor.model_armor import ModelArmorGuardrail
-from litellm.proxy._types import UserAPIKeyAuth
-from litellm.caching.caching import DualCache
-
-@pytest.mark.asyncio
-async def test_model_armor_pre_call_hook_inspect_and_deidentify():
- """
- Test Model Armor guardrail pre-call hook for both inspectResult and deidentifyResult handling.
- """
- guardrail = ModelArmorGuardrail(
- template_id="dummy-template",
- project_id="dummy-project",
- location="us-central1",
- credentials=None,
- )
- armor_response = {
- "sanitizationResult": {
- "filterResults": [
- {
- "sdpFilterResult": {
- "inspectResult": {
- "executionState": "EXECUTION_SUCCESS",
- "matchState": "NO_MATCH_FOUND",
- "findings": []
- },
- "deidentifyResult": {
- "executionState": "EXECUTION_SUCCESS",
- "matchState": "MATCH_FOUND",
- "data": {"text": "sanitized text here"}
- }
- }
- }
- ]
- }
- }
- with patch.object(guardrail, "make_model_armor_request", AsyncMock(return_value=armor_response)):
- user_api_key_dict = UserAPIKeyAuth(api_key="test_key")
- cache = DualCache()
- data = {
- "messages": [
- {"role": "system", "content": "You are a helpful assistant."},
- {"role": "user", "content": "My SSN is 123-45-6789."}
- ],
- "model": "gpt-3.5-turbo",
- "metadata": {}
- }
- guardrail.mask_request_content = True
- with pytest.raises(HTTPException) as exc_info:
- await guardrail.async_pre_call_hook(
- user_api_key_dict=user_api_key_dict,
- cache=cache,
- data=data,
- call_type="completion"
- )
- assert exc_info.value.status_code == 400
- assert "Content blocked by Model Armor" in str(exc_info.value.detail)
-
-def test_model_armor_should_block_content():
- guardrail = ModelArmorGuardrail(
- template_id="dummy-template",
- project_id="dummy-project",
- location="us-central1",
- credentials=None,
- )
- # Block on inspectResult
- armor_response_inspect = {
- "sanitizationResult": {
- "filterResults": [
- {"sdpFilterResult": {"inspectResult": {"matchState": "MATCH_FOUND"}}}
- ]
- }
- }
- assert guardrail._should_block_content(armor_response_inspect)
- # Block on deidentifyResult
- armor_response_deidentify = {
- "sanitizationResult": {
- "filterResults": [
- {"sdpFilterResult": {"deidentifyResult": {"matchState": "MATCH_FOUND"}}}
- ]
- }
- }
- assert guardrail._should_block_content(armor_response_deidentify)
- # No block if neither
- armor_response_none = {
- "sanitizationResult": {
- "filterResults": [
- {"sdpFilterResult": {"inspectResult": {"matchState": "NO_MATCH_FOUND"}, "deidentifyResult": {"matchState": "NO_MATCH_FOUND"}}}
- ]
- }
- }
- assert not guardrail._should_block_content(armor_response_none)
diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py
index 45702a261e2..307c429fc62 100644
--- a/tests/llm_translation/test_anthropic_completion.py
+++ b/tests/llm_translation/test_anthropic_completion.py
@@ -256,7 +256,6 @@ def test_anthropic_tool_streaming():
for chunk in anthropic_chunk_list:
parsed_chunk = response_iter.chunk_parser(chunk)
if tool_use := parsed_chunk.get("tool_use"):
-
# We only increment when a new block starts
if tool_use.get("id") is not None:
correct_tool_index += 1
@@ -920,6 +919,14 @@ def test_anthropic_citations_api():
citations = resp.choices[0].message.provider_specific_fields["citations"]
assert citations is not None
+ if citations:
+ citation = citations[0][0]
+ assert "supported_text" in citation
+ assert "cited_text" in citation
+ assert "document_index" in citation
+ assert "document_title" in citation
+ assert "start_char_index" in citation
+ assert "end_char_index" in citation
def test_anthropic_citations_api_streaming():
@@ -955,11 +962,9 @@ def test_anthropic_citations_api_streaming():
has_citations = False
for chunk in resp:
print(f"returned chunk: {chunk}")
- if (
- chunk.choices[0].delta.provider_specific_fields
- and "citation" in chunk.choices[0].delta.provider_specific_fields
- ):
- has_citations = True
+ if provider_specific_fields := chunk.choices[0].delta.provider_specific_fields:
+ if "citation" in provider_specific_fields:
+ has_citations = True
assert has_citations
diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py
index af76c82685e..2aae51110d3 100644
--- a/tests/local_testing/test_amazing_vertex_completion.py
+++ b/tests/local_testing/test_amazing_vertex_completion.py
@@ -2326,63 +2326,6 @@ def test_prompt_factory_nested():
), "'text' value not a string."
-def test_get_token_url():
- from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
- VertexLLM,
- )
-
- vertex_llm = VertexLLM()
- vertex_ai_project = "pathrise-convert-1606954137718"
- vertex_ai_location = "us-central1"
- json_obj = get_vertex_ai_creds_json()
- vertex_credentials = json.dumps(json_obj)
-
- should_use_v1beta1_features = vertex_llm.is_using_v1beta1_features(
- optional_params={"cached_content": "hi"}
- )
-
- assert should_use_v1beta1_features is True
-
- _, url = vertex_llm._get_token_and_url(
- auth_header=None,
- vertex_project=vertex_ai_project,
- vertex_location=vertex_ai_location,
- vertex_credentials=vertex_credentials,
- gemini_api_key="",
- custom_llm_provider="vertex_ai_beta",
- should_use_v1beta1_features=should_use_v1beta1_features,
- api_base=None,
- model="",
- stream=False,
- )
-
- print("url=", url)
-
- assert "/v1beta1/" in url
-
- should_use_v1beta1_features = vertex_llm.is_using_v1beta1_features(
- optional_params={"temperature": 0.1}
- )
-
- _, url = vertex_llm._get_token_and_url(
- auth_header=None,
- vertex_project=vertex_ai_project,
- vertex_location=vertex_ai_location,
- vertex_credentials=vertex_credentials,
- gemini_api_key="",
- custom_llm_provider="vertex_ai_beta",
- should_use_v1beta1_features=should_use_v1beta1_features,
- api_base=None,
- model="",
- stream=False,
- )
-
- print("url for normal request", url)
-
- assert "v1beta1" not in url
- assert "/v1/" in url
-
- pass
@pytest.mark.asyncio
diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py
index 390ea5835ca..9fc370fdcce 100644
--- a/tests/local_testing/test_completion.py
+++ b/tests/local_testing/test_completion.py
@@ -4321,20 +4321,6 @@ def test_langfuse_completion(monkeypatch):
)
-def test_humanloop_completion(monkeypatch):
- monkeypatch.setenv(
- "HUMANLOOP_API_KEY", "hl_sk_59c1206e110c3f5b9985f0de4d23e7cbc79c4c4ae18c9f14"
- )
- litellm.set_verbose = True
- resp = litellm.completion(
- model="humanloop/gpt-3.5-turbo",
- humanloop_api_key=os.getenv("HUMANLOOP_API_KEY"),
- prompt_id="pr_nmSOVpEdyYPm2DrOwCoOm",
- prompt_variables={"person": "John"},
- messages=[{"role": "user", "content": "Tell me a joke."}],
- )
-
-
def test_completion_novita_ai():
litellm.set_verbose = True
messages = [
diff --git a/tests/test_litellm/integrations/test_langfuse.py b/tests/test_litellm/integrations/test_langfuse.py
index 6cf7b98c2e6..fa2dc3e7190 100644
--- a/tests/test_litellm/integrations/test_langfuse.py
+++ b/tests/test_litellm/integrations/test_langfuse.py
@@ -1,20 +1,222 @@
-import json
-import os
-import sys
-from typing import Optional
-
-# Adds the grandparent directory to sys.path to allow importing project modules
-sys.path.insert(0, os.path.abspath("../.."))
-
+import unittest
import asyncio
-from unittest.mock import patch
-
+from unittest.mock import patch, MagicMock
+from typing import Optional
+import sys
+import os
+import datetime
+import json
import pytest
-
import litellm
from litellm.integrations.langfuse import langfuse as langfuse_module
from litellm.integrations.langfuse.langfuse import LangFuseLogger
+sys.path.insert(0, os.path.abspath("../.."))
+from litellm.integrations.langfuse.langfuse import LangFuseLogger
+# Import LangfuseUsageDetails directly from the module where it's defined
+from litellm.types.integrations.langfuse import *
+
+class TestLangfuseUsageDetails(unittest.TestCase):
+
+ def setUp(self):
+ # Set up environment variables for testing
+ self.env_patcher = patch.dict('os.environ', {
+ 'LANGFUSE_SECRET_KEY': 'test-secret-key',
+ 'LANGFUSE_PUBLIC_KEY': 'test-public-key',
+ 'LANGFUSE_HOST': 'https://test.langfuse.com'
+ })
+ self.env_patcher.start()
+
+ # Create mock objects
+ self.mock_langfuse_client = MagicMock()
+ self.mock_langfuse_trace = MagicMock()
+ self.mock_langfuse_generation = MagicMock()
+
+ # Setup the trace and generation chain
+ self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation
+ self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace
+
+ # Mock the langfuse module that's imported locally in methods
+ self.langfuse_module_patcher = patch.dict('sys.modules', {'langfuse': MagicMock()})
+ self.mock_langfuse_module = self.langfuse_module_patcher.start()
+
+ # Create a mock for the langfuse module with version
+ self.mock_langfuse = MagicMock()
+ self.mock_langfuse.version = MagicMock()
+ self.mock_langfuse.version.__version__ = "3.0.0" # Set a version that supports all features
+
+ # Mock the Langfuse class
+ self.mock_langfuse_class = MagicMock()
+ self.mock_langfuse_class.return_value = self.mock_langfuse_client
+
+ # Set up the sys.modules['langfuse'] mock
+ sys.modules['langfuse'] = self.mock_langfuse
+ sys.modules['langfuse'].Langfuse = self.mock_langfuse_class
+
+ # Mock the Langfuse client
+ self.mock_langfuse_client = MagicMock()
+ self.mock_langfuse_trace = MagicMock()
+ self.mock_langfuse_generation = MagicMock()
+
+ # Setup the trace and generation chain
+ self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation
+ self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace
+
+ # Mock the Langfuse class
+ self.mock_langfuse_class = MagicMock()
+ self.mock_langfuse_class.return_value = self.mock_langfuse_client
+ self.mock_langfuse.Langfuse = self.mock_langfuse_class
+
+ # Create the logger
+ self.logger = LangFuseLogger()
+
+ # Add the log_event_on_langfuse method to the instance
+ def log_event_on_langfuse(self, kwargs, response_obj, start_time=None, end_time=None, user_id=None, level="DEFAULT", status_message=None):
+ # This implementation calls _log_langfuse_v2 directly
+ return self._log_langfuse_v2(
+ user_id=user_id,
+ metadata=kwargs.get("litellm_params", {}).get("metadata", {}),
+ litellm_params=kwargs.get("litellm_params", {}),
+ output=None,
+ start_time=start_time,
+ end_time=end_time,
+ kwargs=kwargs,
+ optional_params=kwargs.get("optional_params", {}),
+ input=None,
+ response_obj=response_obj,
+ level=level,
+ litellm_call_id=kwargs.get("litellm_call_id", None),
+ print_verbose=True # Add the missing parameter
+ )
+
+ # Bind the method to the instance
+ import types
+ self.logger.log_event_on_langfuse = types.MethodType(log_event_on_langfuse, self.logger)
+
+ # Make sure _is_langfuse_v2 returns True
+ def mock_is_langfuse_v2(self):
+ return True
+
+ self.logger._is_langfuse_v2 = types.MethodType(mock_is_langfuse_v2, self.logger)
+
+ def tearDown(self):
+ self.env_patcher.stop()
+ self.langfuse_module_patcher.stop()
+
+ def test_langfuse_usage_details_type(self):
+ """Test that LangfuseUsageDetails TypedDict is properly defined with the correct fields"""
+ # Create an instance of LangfuseUsageDetails
+ usage_details: LangfuseUsageDetails = {
+ "input": 10,
+ "output": 20,
+ "cache_creation_input_tokens": 5,
+ "cache_read_input_tokens": 3
+ }
+
+ # Verify all fields are present
+ self.assertEqual(usage_details["input"], 10)
+ self.assertEqual(usage_details["output"], 20)
+ self.assertEqual(usage_details["cache_creation_input_tokens"], 5)
+ self.assertEqual(usage_details["cache_read_input_tokens"], 3)
+
+ # Test with all fields (all fields are required in TypedDict by default)
+ minimal_usage_details: LangfuseUsageDetails = {
+ "input": 10,
+ "output": 20,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 0
+ }
+
+ self.assertEqual(minimal_usage_details["input"], 10)
+ self.assertEqual(minimal_usage_details["output"], 20)
+
+ def test_log_langfuse_v2_usage_details(self):
+ """Test that usage_details in _log_langfuse_v2 is correctly typed and assigned"""
+ # Create a mock response object with usage information
+ response_obj = MagicMock()
+ response_obj.usage = MagicMock()
+ response_obj.usage.prompt_tokens = 15
+ response_obj.usage.completion_tokens = 25
+
+ # Add the cache token attributes using get method
+ def mock_get(key, default=None):
+ if key == 'cache_creation_input_tokens':
+ return 7
+ elif key == 'cache_read_input_tokens':
+ return 4
+ return default
+
+ response_obj.usage.get = mock_get
+
+ # Create kwargs for the log_event method
+ kwargs = {
+ "model": "gpt-4",
+ "messages": [{"role": "user", "content": "Hello"}],
+ "litellm_params": {"metadata": {}}
+ }
+
+ # Create start and end times
+ start_time = datetime.datetime.now()
+ end_time = start_time + datetime.timedelta(seconds=1)
+
+ # Call the log_event method
+ with patch.object(self.logger, '_log_langfuse_v2') as mock_log_langfuse_v2:
+ self.logger.log_event_on_langfuse(
+ kwargs=kwargs,
+ response_obj=response_obj,
+ start_time=start_time,
+ end_time=end_time
+ )
+
+ # Check if _log_langfuse_v2 was called
+ mock_log_langfuse_v2.assert_called_once()
+
+ # Get the arguments passed to _log_langfuse_v2
+ call_args = mock_log_langfuse_v2.call_args[1]
+
+ # Verify response_obj was passed correctly
+ self.assertEqual(call_args["response_obj"], response_obj)
+
+ def test_langfuse_usage_details_optional_fields(self):
+ """Test that LangfuseUsageDetails fields are properly defined as Optional"""
+ # Create an instance with None values for optional fields
+ usage_details: LangfuseUsageDetails = {
+ "input": 10,
+ "output": 20,
+ "cache_creation_input_tokens": None,
+ "cache_read_input_tokens": None
+ }
+
+ # Verify fields can be None
+ self.assertEqual(usage_details["input"], 10)
+ self.assertEqual(usage_details["output"], 20)
+ self.assertIsNone(usage_details["cache_creation_input_tokens"])
+ self.assertIsNone(usage_details["cache_read_input_tokens"])
+
+ def test_langfuse_usage_details_structure(self):
+ """Test that LangfuseUsageDetails has the correct structure as defined in the commit"""
+ # This test directly verifies the structure of the TypedDict
+ # without relying on the LangFuseLogger class
+
+ # Create a dictionary that matches the LangfuseUsageDetails structure
+ usage_details = {
+ "input": 15,
+ "output": 25,
+ "cache_creation_input_tokens": 7,
+ "cache_read_input_tokens": 4
+ }
+
+ # Verify the structure matches what we expect
+ self.assertIn("input", usage_details)
+ self.assertIn("output", usage_details)
+ self.assertIn("cache_creation_input_tokens", usage_details)
+ self.assertIn("cache_read_input_tokens", usage_details)
+
+ # Verify the values
+ self.assertEqual(usage_details["input"], 15)
+ self.assertEqual(usage_details["output"], 25)
+ self.assertEqual(usage_details["cache_creation_input_tokens"], 7)
+ self.assertEqual(usage_details["cache_read_input_tokens"], 4)
def test_max_langfuse_clients_limit():
"""
diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
index ae44198af11..deb442e8075 100644
--- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
+++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
@@ -183,7 +183,30 @@ def test_extract_response_content_with_citations():
}
_, citations, _, _, _ = config.extract_response_content(completion_response)
- assert citations is not None
+ assert citations == [
+ [
+ {
+ "type": "char_location",
+ "cited_text": "The grass is green. ",
+ "document_index": 0,
+ "document_title": "My Document",
+ "start_char_index": 0,
+ "end_char_index": 20,
+ "supported_text": "the grass is green",
+ },
+ ],
+ [
+ {
+ "type": "char_location",
+ "cited_text": "The sky is blue.",
+ "document_index": 0,
+ "document_title": "My Document",
+ "start_char_index": 20,
+ "end_char_index": 36,
+ "supported_text": "the sky is blue",
+ },
+ ],
+ ]
def test_map_tool_helper():
diff --git a/tests/test_litellm/llms/custom_httpx/test_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_http_handler.py
index a649e9b6b9c..0e31699fd83 100644
--- a/tests/test_litellm/llms/custom_httpx/test_http_handler.py
+++ b/tests/test_litellm/llms/custom_httpx/test_http_handler.py
@@ -29,16 +29,17 @@ async def test_ssl_security_level(monkeypatch):
# Get the transport (should be LiteLLMAiohttpTransport)
transport = client.client._transport
+ assert isinstance(transport, LiteLLMAiohttpTransport)
# Get the aiohttp ClientSession
client_session = transport._get_valid_client_session()
# Get the connector from the session
connector = client_session.connector
+ assert isinstance(connector, TCPConnector)
# Get the SSL context from the connector
ssl_context = connector._ssl
- print("ssl_context", ssl_context)
# Verify that the SSL context exists and has the correct cipher string
assert isinstance(ssl_context, ssl.SSLContext)
@@ -108,20 +109,19 @@ async def test_ssl_verification_with_aiohttp_transport():
# Create a test SSL context
litellm_async_client = AsyncHTTPHandler(ssl_verify=False)
- transport_connector = (
- litellm_async_client.client._transport._get_valid_client_session().connector
- )
- print("transport_connector", transport_connector)
- print("transport_connector._ssl", transport_connector._ssl)
+ transport = litellm_async_client.client._transport
+ assert isinstance(transport, LiteLLMAiohttpTransport)
+ transport_connector = transport._get_valid_client_session().connector
+ assert isinstance(transport_connector, TCPConnector)
aiohttp_session = aiohttp.ClientSession(
connector=aiohttp.TCPConnector(verify_ssl=False)
)
- print("aiohttp_session", aiohttp_session)
- print("aiohttp_session._ssl", aiohttp_session.connector._ssl)
+ aiohttp_connector = aiohttp_session.connector
+ assert isinstance(aiohttp_connector, aiohttp.TCPConnector)
# assert both litellm transport and aiohttp session have ssl_verify=False
- assert transport_connector._ssl == aiohttp_session.connector._ssl
+ assert transport_connector._ssl == aiohttp_connector._ssl
@pytest.mark.asyncio
@@ -183,3 +183,219 @@ def test_get_ssl_configuration_integration():
# Verify it has basic SSL context properties
assert ssl_context.protocol is not None
assert ssl_context.verify_mode is not None
+
+
+# Session Reuse Tests
+class MockClientSession:
+ """Mock ClientSession that is not callable"""
+ def __init__(self):
+ self.closed = False
+
+@pytest.mark.asyncio
+async def test_create_aiohttp_transport_with_shared_session():
+ """Test that _create_aiohttp_transport reuses shared session when provided"""
+ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
+ # Create a mock shared session that's not callable
+ mock_session = MockClientSession()
+
+ # Test with shared session
+ transport = AsyncHTTPHandler._create_aiohttp_transport(
+ shared_session=mock_session # type: ignore
+ )
+
+ # Verify the transport uses the shared session directly
+ assert transport.client is mock_session
+ assert not callable(transport.client) # Should not be callable
+
+
+@pytest.mark.asyncio
+async def test_create_aiohttp_transport_without_shared_session():
+ """Test that _create_aiohttp_transport creates new session when none provided"""
+ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
+ # Test without shared session
+ transport = AsyncHTTPHandler._create_aiohttp_transport(shared_session=None)
+
+ # Verify the transport uses a lambda function (for backward compatibility)
+ assert callable(transport.client) # Should be a lambda function
+
+
+@pytest.mark.asyncio
+async def test_create_aiohttp_transport_with_closed_session():
+ """Test that _create_aiohttp_transport creates new session when shared session is closed"""
+ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
+ # Create a mock closed session
+ mock_session = MockClientSession()
+ mock_session.closed = True
+
+ # Test with closed session
+ transport = AsyncHTTPHandler._create_aiohttp_transport(
+ shared_session=mock_session # type: ignore
+ )
+
+ # Verify the transport creates a new session (lambda function)
+ assert callable(transport.client) # Should be a lambda function
+
+
+@pytest.mark.asyncio
+async def test_async_handler_with_shared_session():
+ """Test AsyncHTTPHandler initialization with shared session"""
+ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
+ # Create a mock shared session
+ mock_session = MockClientSession()
+
+ # Create handler with shared session
+ handler = AsyncHTTPHandler(shared_session=mock_session) # type: ignore
+
+ # Verify the handler was created successfully
+ assert handler is not None
+ assert handler.client is not None
+
+
+@pytest.mark.asyncio
+async def test_get_async_httpx_client_with_shared_session():
+ """Test get_async_httpx_client with shared session"""
+ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+ from litellm.types.utils import LlmProviders
+
+ # Create a mock shared session
+ mock_session = MockClientSession()
+
+ # Test with shared session
+ client = get_async_httpx_client(
+ llm_provider=LlmProviders.ANTHROPIC,
+ shared_session=mock_session # type: ignore
+ )
+
+ # Verify the client was created successfully
+ assert client is not None
+ assert isinstance(client, AsyncHTTPHandler)
+
+
+@pytest.mark.asyncio
+async def test_get_async_httpx_client_without_shared_session():
+ """Test get_async_httpx_client without shared session (backward compatibility)"""
+ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+ from litellm.types.utils import LlmProviders
+
+ # Test without shared session
+ client = get_async_httpx_client(
+ llm_provider=LlmProviders.ANTHROPIC,
+ shared_session=None
+ )
+
+ # Verify the client was created successfully
+ assert client is not None
+ assert isinstance(client, AsyncHTTPHandler)
+
+
+@pytest.mark.asyncio
+async def test_session_reuse_chain():
+ """Test that session is properly passed through the entire call chain"""
+ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
+ # Create a mock shared session
+ mock_session = MockClientSession()
+
+ # Test the entire chain
+ transport = AsyncHTTPHandler._create_async_transport(
+ shared_session=mock_session # type: ignore
+ )
+
+ # Verify the transport was created
+ assert transport is not None
+
+ # Test AsyncHTTPHandler creation
+ handler = AsyncHTTPHandler(shared_session=mock_session) # type: ignore
+ assert handler is not None
+
+
+def test_shared_session_parameter_in_acompletion():
+ """Test that acompletion function accepts shared_session parameter"""
+ import inspect
+ from litellm.main import acompletion
+
+ # Get the function signature
+ sig = inspect.signature(acompletion)
+ params = list(sig.parameters.keys())
+
+ # Verify shared_session parameter exists
+ assert 'shared_session' in params
+
+ # Verify the parameter type annotation
+ shared_session_param = sig.parameters['shared_session']
+ assert 'ClientSession' in str(shared_session_param.annotation)
+
+
+def test_shared_session_parameter_in_completion():
+ """Test that completion function accepts shared_session parameter"""
+ import inspect
+ from litellm.main import completion
+
+ # Get the function signature
+ sig = inspect.signature(completion)
+ params = list(sig.parameters.keys())
+
+ # Verify shared_session parameter exists
+ assert 'shared_session' in params
+
+ # Verify the parameter type annotation
+ shared_session_param = sig.parameters['shared_session']
+ assert 'ClientSession' in str(shared_session_param.annotation)
+
+
+@pytest.mark.asyncio
+async def test_session_reuse_integration():
+ """Integration test for session reuse functionality"""
+ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+ from litellm.types.utils import LlmProviders
+
+ # Create a mock session
+ mock_session = MockClientSession()
+
+ # Create two clients with the same session
+ client1 = get_async_httpx_client(
+ llm_provider=LlmProviders.ANTHROPIC,
+ shared_session=mock_session # type: ignore
+ )
+
+ client2 = get_async_httpx_client(
+ llm_provider=LlmProviders.OPENAI,
+ shared_session=mock_session # type: ignore
+ )
+
+ # Both clients should be created successfully
+ assert client1 is not None
+ assert client2 is not None
+
+ # Both should be AsyncHTTPHandler instances
+ assert isinstance(client1, AsyncHTTPHandler)
+ assert isinstance(client2, AsyncHTTPHandler)
+
+ # Clean up
+ await client1.close()
+ await client2.close()
+
+
+@pytest.mark.asyncio
+async def test_session_validation():
+ """Test that session validation works correctly"""
+ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
+ # Test with None session
+ transport1 = AsyncHTTPHandler._create_aiohttp_transport(shared_session=None)
+ assert callable(transport1.client) # Should create lambda
+
+ # Test with closed session
+ mock_closed_session = MockClientSession()
+ mock_closed_session.closed = True
+ transport2 = AsyncHTTPHandler._create_aiohttp_transport(shared_session=mock_closed_session) # type: ignore
+ assert callable(transport2.client) # Should create lambda
+
+ # Test with valid session
+ mock_valid_session = MockClientSession()
+ transport3 = AsyncHTTPHandler._create_aiohttp_transport(shared_session=mock_valid_session) # type: ignore
+ assert transport3.client is mock_valid_session # Should reuse session
diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
index 02cd51920da..02dac0a93d6 100644
--- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
+++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
@@ -801,3 +801,59 @@ def test_fix_enum_empty_strings():
# 3. Other properties preserved
assert input_schema["properties"]["user_agent_type"]["type"] == "string"
assert input_schema["properties"]["user_agent_type"]["description"] == "Device type for user agent"
+
+
+def test_get_token_url():
+ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
+ VertexLLM,
+ )
+
+ vertex_llm = VertexLLM()
+ vertex_ai_project = "pathrise-convert-1606954137718"
+ vertex_ai_location = "us-central1"
+ vertex_credentials = ""
+
+ should_use_v1beta1_features = vertex_llm.is_using_v1beta1_features(
+ optional_params={"cached_content": "hi"}
+ )
+
+ _, url = vertex_llm._get_token_and_url(
+ auth_header=None,
+ vertex_project=vertex_ai_project,
+ vertex_location=vertex_ai_location,
+ vertex_credentials=vertex_credentials,
+ gemini_api_key="",
+ custom_llm_provider="vertex_ai_beta",
+ should_use_v1beta1_features=should_use_v1beta1_features,
+ api_base=None,
+ model="",
+ stream=False,
+ )
+
+ print("url=", url)
+
+
+
+ should_use_v1beta1_features = vertex_llm.is_using_v1beta1_features(
+ optional_params={"temperature": 0.1}
+ )
+
+ _, url = vertex_llm._get_token_and_url(
+ auth_header=None,
+ vertex_project=vertex_ai_project,
+ vertex_location=vertex_ai_location,
+ vertex_credentials=vertex_credentials,
+ gemini_api_key="",
+ custom_llm_provider="vertex_ai_beta",
+ should_use_v1beta1_features=should_use_v1beta1_features,
+ api_base=None,
+ model="",
+ stream=False,
+ )
+
+ print("url for normal request", url)
+
+ assert "v1beta1" not in url
+ assert "/v1/" in url
+
+ pass
\ No newline at end of file
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
index 754487d11e7..8fa9964b1fc 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py
@@ -102,7 +102,7 @@ async def test_get_tools_from_mcp_servers_continues_when_one_server_fails():
working_server if server_id == "working_server" else failing_server
)
- async def mock_get_tools_from_server(server, mcp_auth_header=None, add_prefix=True):
+ async def mock_get_tools_from_server(server, mcp_auth_header=None):
if server.name == "working_server":
# Working server returns tools
tool1 = MagicMock()
@@ -184,7 +184,7 @@ async def test_get_tools_from_mcp_servers_handles_all_servers_failing():
failing_server1 if server_id == "failing_server1" else failing_server2
)
- async def mock_get_tools_from_server(server, mcp_auth_header=None, add_prefix=True):
+ async def mock_get_tools_from_server(server, mcp_auth_header=None):
# All servers fail
raise Exception(f"Server {server.name} connection failed")
@@ -448,121 +448,3 @@ async def test_mcp_routing_with_conflicting_alias_and_group_name():
assert (
called_servers[0].server_id == specific_server.server_id
), "Should have contacted the specific server alias, not the group."
-
-
-@pytest.mark.asyncio
-async def test_list_tools_single_server_unprefixed_names():
- """When only one MCP server is allowed, list tools should return unprefixed names."""
- try:
- from litellm.proxy._experimental.mcp_server.server import (
- _get_tools_from_mcp_servers,
- set_auth_context,
- )
- except ImportError:
- pytest.skip("MCP server not available")
-
- # Mock user auth
- user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user")
- set_auth_context(user_api_key_auth)
-
- # One allowed server
- server = MagicMock()
- server.server_id = "server1"
- server.name = "Zapier MCP"
- server.alias = "zapier"
-
- # Mock manager: allow just one server and return a tool based on add_prefix flag
- mock_manager = MagicMock()
- mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=["server1"])
- mock_manager.get_mcp_server_by_id = (
- lambda server_id: server if server_id == "server1" else None
- )
-
- async def mock_get_tools_from_server(
- server, mcp_auth_header=None, add_prefix=False
- ):
- tool = MagicMock()
- tool.name = f"{server.alias}-toolA" if add_prefix else "toolA"
- tool.description = "desc"
- tool.inputSchema = {}
- return [tool]
-
- mock_manager._get_tools_from_server = mock_get_tools_from_server
-
- with patch(
- "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
- mock_manager,
- ):
- tools = await _get_tools_from_mcp_servers(
- user_api_key_auth=user_api_key_auth,
- mcp_auth_header=None,
- mcp_servers=None,
- mcp_server_auth_headers=None,
- )
-
- # Should be unprefixed since only one server is allowed
- assert len(tools) == 1
- assert tools[0].name == "toolA"
-
-
-@pytest.mark.asyncio
-async def test_list_tools_multiple_servers_prefixed_names():
- """When multiple MCP servers are allowed, list tools should return prefixed names."""
- try:
- from litellm.proxy._experimental.mcp_server.server import (
- _get_tools_from_mcp_servers,
- set_auth_context,
- )
- except ImportError:
- pytest.skip("MCP server not available")
-
- # Mock user auth
- user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user")
- set_auth_context(user_api_key_auth)
-
- # Two allowed servers
- server1 = MagicMock()
- server1.server_id = "server1"
- server1.name = "Zapier MCP"
- server1.alias = "zapier"
-
- server2 = MagicMock()
- server2.server_id = "server2"
- server2.name = "Jira MCP"
- server2.alias = "jira"
-
- # Mock manager
- mock_manager = MagicMock()
- mock_manager.get_allowed_mcp_servers = AsyncMock(
- return_value=["server1", "server2"]
- )
- mock_manager.get_mcp_server_by_id = (
- lambda server_id: server1 if server_id == "server1" else server2
- )
-
- async def mock_get_tools_from_server(
- server, mcp_auth_header=None, add_prefix=True
- ):
- tool = MagicMock()
- # When multiple servers, add_prefix should be True -> prefixed names
- tool.name = f"{server.alias}-toolA" if add_prefix else "toolA"
- tool.description = "desc"
- tool.inputSchema = {}
- return [tool]
-
- mock_manager._get_tools_from_server = mock_get_tools_from_server
-
- with patch(
- "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
- mock_manager,
- ):
- tools = await _get_tools_from_mcp_servers(
- user_api_key_auth=user_api_key_auth,
- mcp_auth_header=None,
- mcp_servers=None,
- mcp_server_auth_headers=None,
- )
-
- # Should be prefixed since multiple servers are allowed
- names = sorted([t.name for t in tools])
- assert names == ["jira-toolA", "zapier-toolA"]
diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
index e3bb085d5f4..3237de37636 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
@@ -420,112 +420,6 @@ class TestMCPServerManager:
assert result["status"] == "healthy"
assert result["tools_count"] == 1
- @pytest.mark.asyncio
- async def test_get_tools_from_server_add_prefix(self):
- """Verify _get_tools_from_server respects add_prefix True/False."""
- manager = MCPServerManager()
-
- # Create a minimal server with alias used as prefix
- server = MCPServer(
- server_id="zapier",
- name="zapier",
- transport=MCPTransport.http,
- )
-
- # Mock client creation and fetching tools
- manager._create_mcp_client = MagicMock(return_value=object())
-
- # Tools returned upstream (unprefixed from provider)
- upstream_tool = MagicMock()
- upstream_tool.name = "send_email"
- upstream_tool.description = "Send an email"
- upstream_tool.inputSchema = {}
-
- manager._fetch_tools_with_timeout = AsyncMock(return_value=[upstream_tool])
-
- # Case 1: add_prefix=True (default for multi-server) -> expect prefixed
- tools_prefixed = await manager._get_tools_from_server(server, add_prefix=True)
- assert len(tools_prefixed) == 1
- assert tools_prefixed[0].name == "zapier-send_email"
-
- # Case 2: add_prefix=False (single-server) -> expect unprefixed
- tools_unprefixed = await manager._get_tools_from_server(
- server, add_prefix=False
- )
- assert len(tools_unprefixed) == 1
- assert tools_unprefixed[0].name == "send_email"
-
- def test_create_prefixed_tools_updates_mapping_for_both_forms(self):
- """_create_prefixed_tools should populate mapping for prefixed and original names even when not adding prefix in output."""
- manager = MCPServerManager()
-
- server = MCPServer(
- server_id="jira",
- name="jira",
- transport=MCPTransport.http,
- )
-
- # Input tools as would come from upstream
- t1 = MagicMock()
- t1.name = "create_issue"
- t1.description = ""
- t1.inputSchema = {}
- t2 = MagicMock()
- t2.name = "close_issue"
- t2.description = ""
- t2.inputSchema = {}
-
- # Do not add prefix in returned objects
- out_tools = manager._create_prefixed_tools([t1, t2], server, add_prefix=False)
-
- # Returned names should be unprefixed
- names = sorted([t.name for t in out_tools])
- assert names == ["close_issue", "create_issue"]
-
- # Mapping should include both original and prefixed names -> resolves calls either way
- assert manager.tool_name_to_mcp_server_name_mapping["create_issue"] == "jira"
- assert (
- manager.tool_name_to_mcp_server_name_mapping["jira-create_issue"] == "jira"
- )
- assert manager.tool_name_to_mcp_server_name_mapping["close_issue"] == "jira"
- assert (
- manager.tool_name_to_mcp_server_name_mapping["jira-close_issue"] == "jira"
- )
-
- def test_get_mcp_server_from_tool_name_with_prefixed_and_unprefixed(self):
- """After mapping is populated, manager resolves both prefixed and unprefixed tool names to the same server."""
- manager = MCPServerManager()
-
- server = MCPServer(
- server_id="zapier",
- name="zapier",
- server_name="zapier",
- transport=MCPTransport.http,
- )
-
- # Register server so resolution can find it
- manager.registry = {server.server_id: server}
-
- # Populate mapping (add_prefix value doesn't matter for mapping population)
- base_tool = MagicMock()
- base_tool.name = "create_zap"
- base_tool.description = ""
- base_tool.inputSchema = {}
- _ = manager._create_prefixed_tools([base_tool], server, add_prefix=False)
-
- # Unprefixed resolution
- resolved_server_unpref = manager._get_mcp_server_from_tool_name("create_zap")
- print(resolved_server_unpref)
- assert resolved_server_unpref is not None
- assert resolved_server_unpref.server_id == server.server_id
-
- # Prefixed resolution
- resolved_server_pref = manager._get_mcp_server_from_tool_name(
- "zapier-create_zap"
- )
- assert resolved_server_pref is not None
- assert resolved_server_pref.server_id == server.server_id
-
if __name__ == "__main__":
pytest.main([__file__])
diff --git a/tests/test_litellm/proxy/client/cli/test_auth_commands.py b/tests/test_litellm/proxy/client/cli/test_auth_commands.py
index 5ef96e4f9af..0ea81c70eb1 100644
--- a/tests/test_litellm/proxy/client/cli/test_auth_commands.py
+++ b/tests/test_litellm/proxy/client/cli/test_auth_commands.py
@@ -159,13 +159,13 @@ class TestTokenUtilities:
'user_id': 'test-user'
}
- with patch('litellm.proxy.client.cli.commands.auth.load_token', return_value=token_data):
+ with patch('litellm.litellm_core_utils.cli_token_utils.load_cli_token', return_value=token_data):
result = get_stored_api_key()
assert result == 'test-api-key-123'
def test_get_stored_api_key_no_token(self):
"""Test getting stored API key when no token exists"""
- with patch('litellm.proxy.client.cli.commands.auth.load_token', return_value=None):
+ with patch('litellm.litellm_core_utils.cli_token_utils.load_cli_token', return_value=None):
result = get_stored_api_key()
assert result is None
@@ -175,7 +175,7 @@ class TestTokenUtilities:
'user_id': 'test-user'
}
- with patch('litellm.proxy.client.cli.commands.auth.load_token', return_value=token_data):
+ with patch('litellm.litellm_core_utils.cli_token_utils.load_cli_token', return_value=token_data):
result = get_stored_api_key()
assert result is None
@@ -471,7 +471,7 @@ class TestCLIKeyRegenerationFlow:
assert result.exit_code == 0
assert "✅ Login successful!" in result.output
- assert "API Key: sk-regenerated-key-456" in result.output
+ assert "API Key: sk-regenerated-key-4..." in result.output
# Verify existing key was retrieved
mock_get_stored.assert_called_once()
@@ -486,7 +486,9 @@ class TestCLIKeyRegenerationFlow:
# Verify polling was done with correct session key
mock_get.assert_called()
- poll_url = mock_get.call_args[0][0]
+ # Check that the polling URL was called (should be the first call)
+ first_call_args = mock_get.call_args_list[0]
+ poll_url = first_call_args[0][0]
assert "sk-new-session-uuid-789" in poll_url
# Verify regenerated key was saved
diff --git a/tests/test_litellm/proxy/client/cli/test_chat_commands.py b/tests/test_litellm/proxy/client/cli/test_chat_commands.py
deleted file mode 100644
index 1226c3557ff..00000000000
--- a/tests/test_litellm/proxy/client/cli/test_chat_commands.py
+++ /dev/null
@@ -1,248 +0,0 @@
-import json
-import os
-import sys
-from unittest.mock import MagicMock, patch
-
-import pytest
-import requests
-from click.testing import CliRunner
-
-sys.path.insert(
- 0, os.path.abspath("../../..")
-) # Adds the parent directory to the system path
-
-
-from litellm.proxy.client.cli.main import cli
-
-
-@pytest.fixture
-def mock_chat_client():
- with patch("litellm.proxy.client.cli.commands.chat.ChatClient") as mock:
- yield mock
-
-
-@pytest.fixture
-def cli_runner():
- return CliRunner()
-
-
-def test_chat_completions_success(cli_runner, mock_chat_client):
- # Mock response data
- mock_response = {
- "id": "chatcmpl-123",
- "object": "chat.completion",
- "created": 1677858242,
- "model": "gpt-4",
- "choices": [
- {
- "message": {
- "role": "assistant",
- "content": "Hello! How can I help you today?",
- },
- "finish_reason": "stop",
- "index": 0,
- }
- ],
- }
- mock_instance = mock_chat_client.return_value
- mock_instance.completions.return_value = mock_response
-
- # Run command
- result = cli_runner.invoke(
- cli,
- [
- "chat",
- "completions",
- "gpt-4",
- "-m",
- "user:Hello!",
- "--temperature",
- "0.7",
- "--max-tokens",
- "100",
- ],
- )
-
- # Verify
- assert result.exit_code == 0
- output_data = json.loads(result.output)
- assert output_data == mock_response
- mock_instance.completions.assert_called_once_with(
- model="gpt-4",
- messages=[{"role": "user", "content": "Hello!"}],
- temperature=0.7,
- max_tokens=100,
- top_p=None,
- n=None,
- presence_penalty=None,
- frequency_penalty=None,
- user=None,
- )
-
-
-def test_chat_completions_multiple_messages(cli_runner, mock_chat_client):
- # Mock response data
- mock_response = {
- "id": "chatcmpl-123",
- "object": "chat.completion",
- "created": 1677858242,
- "model": "gpt-4",
- "choices": [
- {
- "message": {
- "role": "assistant",
- "content": "Paris has a population of about 2.2 million.",
- },
- "finish_reason": "stop",
- "index": 0,
- }
- ],
- }
- mock_instance = mock_chat_client.return_value
- mock_instance.completions.return_value = mock_response
-
- # Run command
- result = cli_runner.invoke(
- cli,
- [
- "chat",
- "completions",
- "gpt-4",
- "-m",
- "system:You are a helpful assistant",
- "-m",
- "user:What's the population of Paris?",
- ],
- )
-
- # Verify
- assert result.exit_code == 0
- output_data = json.loads(result.output)
- assert output_data == mock_response
- mock_instance.completions.assert_called_once_with(
- model="gpt-4",
- messages=[
- {"role": "system", "content": "You are a helpful assistant"},
- {"role": "user", "content": "What's the population of Paris?"},
- ],
- temperature=None,
- max_tokens=None,
- top_p=None,
- n=None,
- presence_penalty=None,
- frequency_penalty=None,
- user=None,
- )
-
-
-def test_chat_completions_no_messages(cli_runner, mock_chat_client):
- # Run command without any messages
- result = cli_runner.invoke(cli, ["chat", "completions", "gpt-4"])
-
- # Verify
- assert result.exit_code == 2
- assert "At least one message is required" in result.output
- mock_instance = mock_chat_client.return_value
- mock_instance.completions.assert_not_called()
-
-
-def test_chat_completions_invalid_message_format(cli_runner, mock_chat_client):
- # Run command with invalid message format
- result = cli_runner.invoke(
- cli, ["chat", "completions", "gpt-4", "-m", "invalid-format"]
- )
-
- # Verify
- assert result.exit_code == 2
- assert "Invalid message format" in result.output
- mock_instance = mock_chat_client.return_value
- mock_instance.completions.assert_not_called()
-
-
-def test_chat_completions_http_error(cli_runner, mock_chat_client):
- # Mock HTTP error
- mock_instance = mock_chat_client.return_value
- mock_error_response = MagicMock()
- mock_error_response.status_code = 400
- mock_error_response.json.return_value = {
- "error": "Invalid request",
- "message": "Invalid model specified",
- }
- mock_instance.completions.side_effect = requests.exceptions.HTTPError(
- response=mock_error_response
- )
-
- # Run command
- result = cli_runner.invoke(
- cli, ["chat", "completions", "invalid-model", "-m", "user:Hello"]
- )
-
- # Verify
- assert result.exit_code == 1
- assert "Error: HTTP 400" in result.output
- assert "Invalid request" in result.output
- assert "Invalid model specified" in result.output
-
-
-def test_chat_completions_all_parameters(cli_runner, mock_chat_client):
- # Mock response data
- mock_response = {
- "id": "chatcmpl-123",
- "object": "chat.completion",
- "created": 1677858242,
- "model": "gpt-4",
- "choices": [
- {
- "message": {
- "role": "assistant",
- "content": "Response with all parameters set",
- },
- "finish_reason": "stop",
- "index": 0,
- }
- ],
- }
- mock_instance = mock_chat_client.return_value
- mock_instance.completions.return_value = mock_response
-
- # Run command with all available parameters
- result = cli_runner.invoke(
- cli,
- [
- "chat",
- "completions",
- "gpt-4",
- "-m",
- "user:Test message",
- "--temperature",
- "0.7",
- "--top-p",
- "0.9",
- "--n",
- "1",
- "--max-tokens",
- "100",
- "--presence-penalty",
- "0.5",
- "--frequency-penalty",
- "0.5",
- "--user",
- "test-user",
- ],
- )
-
- # Verify
- assert result.exit_code == 0
- output_data = json.loads(result.output)
- assert output_data == mock_response
- mock_instance.completions.assert_called_once_with(
- model="gpt-4",
- messages=[{"role": "user", "content": "Test message"}],
- temperature=0.7,
- top_p=0.9,
- n=1,
- max_tokens=100,
- presence_penalty=0.5,
- frequency_penalty=0.5,
- user="test-user",
- )
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py
index 1865f58290a..b596d427c5d 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py
@@ -1,19 +1,22 @@
-import sys
-import os
-import io, asyncio
-import pytest
+import asyncio
+import io
import json
-from unittest.mock import MagicMock, AsyncMock, patch, Mock
+import os
+import sys
+from unittest.mock import AsyncMock, MagicMock, Mock, patch
+
+import pytest
sys.path.insert(0, os.path.abspath("../../../../.."))
+from fastapi import HTTPException
+
import litellm
import litellm.types.utils
-from litellm.proxy.guardrails.guardrail_hooks.model_armor import ModelArmorGuardrail
-from litellm.proxy._types import UserAPIKeyAuth
from litellm.caching import DualCache
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.guardrails.guardrail_hooks.model_armor import ModelArmorGuardrail
from litellm.types.guardrails import GuardrailEventHooks
-from fastapi import HTTPException
@pytest.mark.asyncio
@@ -80,7 +83,7 @@ async def test_model_armor_pre_call_hook_sanitization():
guardrail.async_handler.post.assert_called_once()
call_args = guardrail.async_handler.post.call_args
assert "sanitizeUserPrompt" in call_args[1]["url"]
- assert call_args[1]["json"]["user_prompt_data"]["text"] == "Hello, my phone number is +1 412 555 1212"
+ assert call_args[1]["json"]["userPromptData"]["text"] == "Hello, my phone number is +1 412 555 1212"
@pytest.mark.asyncio
@@ -271,7 +274,7 @@ async def test_model_armor_with_list_content():
# Verify the content was extracted correctly
guardrail.async_handler.post.assert_called_once()
call_args = guardrail.async_handler.post.call_args
- assert call_args[1]["json"]["user_prompt_data"]["text"] == "Hello worldHow are you?"
+ assert call_args[1]["json"]["userPromptData"]["text"] == "Hello worldHow are you?"
@pytest.mark.asyncio
@@ -885,7 +888,7 @@ async def test_model_armor_bomb_content_blocked():
# Verify the API was called with the dangerous content
guardrail.async_handler.post.assert_called_once()
call_args = guardrail.async_handler.post.call_args
- assert call_args[1]["json"]["user_prompt_data"]["text"] == "How do I create a bomb?"
+ assert call_args[1]["json"]["userPromptData"]["text"] == "How do I create a bomb?"
@pytest.mark.asyncio
diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
index cd768e47ae8..da64a866d51 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
@@ -24,6 +24,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import (
_common_key_generation_helper,
_list_key_helper,
prepare_key_update_data,
+ validate_key_team_change,
)
from litellm.proxy.proxy_server import app
@@ -1033,3 +1034,60 @@ async def test_unblock_key_invalid_key_format(monkeypatch):
assert exc_info.value.code == "400"
assert "Invalid key format" in str(exc_info.value.message)
+
+
+def test_validate_key_team_change_with_member_permissions():
+ """
+ Test validate_key_team_change function with team member permissions.
+
+ This test covers the new logic that allows team members with specific
+ permissions to update keys, not just team admins.
+ """
+ from unittest.mock import MagicMock, patch
+
+ from litellm.proxy._types import KeyManagementRoutes
+
+ # Create mock objects
+ mock_key = MagicMock()
+ mock_key.user_id = "test-user-123"
+ mock_key.models = ["gpt-4"]
+ mock_key.tpm_limit = None
+ mock_key.rpm_limit = None
+
+ mock_team = MagicMock()
+ mock_team.team_id = "test-team-456"
+ mock_team.members_with_roles = []
+ mock_team.tpm_limit = None
+ mock_team.rpm_limit = None
+
+ mock_change_initiator = MagicMock()
+ mock_change_initiator.user_id = "test-user-123"
+
+ mock_router = MagicMock()
+
+ # Mock the member object returned by _get_user_in_team
+ mock_member_object = MagicMock()
+
+ with patch('litellm.proxy.management_endpoints.key_management_endpoints.can_team_access_model'):
+ with patch('litellm.proxy.management_endpoints.key_management_endpoints._get_user_in_team') as mock_get_user:
+ with patch('litellm.proxy.management_endpoints.key_management_endpoints._is_user_team_admin') as mock_is_admin:
+ with patch('litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.does_team_member_have_permissions_for_endpoint') as mock_has_perms:
+
+ mock_get_user.return_value = mock_member_object
+ mock_is_admin.return_value = False
+ mock_has_perms.return_value = True
+
+ # This should not raise an exception due to member permissions
+ validate_key_team_change(
+ key=mock_key,
+ team=mock_team,
+ change_initiated_by=mock_change_initiator,
+ llm_router=mock_router
+ )
+
+ # Verify the permission check was called with correct parameters
+ mock_has_perms.assert_called_once_with(
+ team_member_object=mock_member_object,
+ team_table=mock_team,
+ route=KeyManagementRoutes.KEY_UPDATE.value
+ )
diff --git a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
index fc97b389137..f1594c36e7e 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py
@@ -22,7 +22,7 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.types.mcp import MCPAuth
-from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer
+from litellm.types.mcp_server.mcp_server_manager import MCPServer
def generate_mock_mcp_server_db_record(
@@ -63,10 +63,10 @@ def generate_mock_mcp_server_config_record(
url=url,
transport=MCPTransport.http if transport == "http" else MCPTransport.sse,
auth_type=MCPAuth.api_key if auth_type == "api_key" else None,
- mcp_info=MCPInfo(
- server_name=name,
- description="Config server description",
- ),
+ mcp_info={
+ "server_name": name,
+ "description": "Config server description",
+ },
)
diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py
index 80aebc98497..639b4756533 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py
@@ -543,6 +543,7 @@ async def test_get_user_info_from_db():
assert mock_get_user_object.call_args.kwargs["user_id"] == "krrishd"
+@pytest.mark.asyncio
async def test_get_user_info_from_db_alternate_user_id():
from litellm.proxy.management_endpoints.ui_sso import get_user_info_from_db
@@ -1275,7 +1276,7 @@ class TestCLIKeyRegenerationFlow:
)
# Assert
- mock_regenerate.assert_called_once_with(existing_key, new_key)
+ mock_regenerate.assert_called_once_with(existing_key=existing_key, new_key=new_key, user_id=None)
assert result.status_code == 200
assert "Success" in result.body.decode()
@@ -1303,7 +1304,7 @@ class TestCLIKeyRegenerationFlow:
)
# Assert
- mock_create.assert_called_once_with(new_key)
+ mock_create.assert_called_once_with(key=new_key, user_id=None)
assert result.status_code == 200
assert "Success" in result.body.decode()
@@ -1320,8 +1321,17 @@ class TestCLIKeyRegenerationFlow:
# CLI state
cli_state = f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:sk-new-session-key-456"
- # Mock the CLI callback
- with patch("litellm.proxy.management_endpoints.ui_sso.cli_sso_callback") as mock_cli_callback:
+ # Mock the CLI callback and required proxy server components
+ mock_result = {"user_id": "test-user", "email": "test@example.com"}
+
+ with patch("litellm.proxy.management_endpoints.ui_sso.cli_sso_callback") as mock_cli_callback, \
+ patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), \
+ patch("litellm.proxy.proxy_server.master_key", "test-master-key"), \
+ patch("litellm.proxy.proxy_server.general_settings", {}), \
+ patch("litellm.proxy.proxy_server.jwt_handler", MagicMock()), \
+ patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), \
+ patch.dict(os.environ, {"GOOGLE_CLIENT_ID": "test-google-id"}, clear=True), \
+ patch("litellm.proxy.management_endpoints.ui_sso.GoogleSSOHandler.get_google_callback_response", return_value=mock_result):
mock_cli_callback.return_value = MagicMock()
# Act
@@ -1329,9 +1339,10 @@ class TestCLIKeyRegenerationFlow:
# Assert
mock_cli_callback.assert_called_once_with(
- mock_request,
+ request=mock_request,
key="sk-new-session-key-456",
- existing_key="sk-existing-cli-key-123"
+ existing_key="sk-existing-cli-key-123",
+ result=mock_result
)
def test_get_redirect_url_preserves_existing_key(self):
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_anthropic_auth_headers.py b/tests/test_litellm/proxy/pass_through_endpoints/test_anthropic_auth_headers.py
new file mode 100644
index 00000000000..9872ed2c9d1
--- /dev/null
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_anthropic_auth_headers.py
@@ -0,0 +1,218 @@
+import os
+import sys
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+sys.path.insert(
+ 0, os.path.abspath("../../..")
+) # Adds the parent directory to the system path
+
+from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
+ anthropic_proxy_route,
+)
+
+
+class TestAnthropicAuthHeaders:
+ """Test authentication header handling in anthropic_proxy_route."""
+
+ @pytest.fixture
+ def mock_request(self):
+ """Create a mock request object."""
+ request = MagicMock()
+ request.method = "POST"
+ request.headers = {}
+ return request
+
+ @pytest.fixture
+ def mock_response(self):
+ """Create a mock FastAPI response object."""
+ return MagicMock()
+
+ @pytest.fixture
+ def mock_user_api_key_dict(self):
+ """Create a mock user API key dict."""
+ return {"user_id": "test_user"}
+
+ @pytest.mark.asyncio
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route")
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_streaming_request_fn")
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router")
+ async def test_client_authorization_header_priority(
+ self,
+ mock_router,
+ mock_streaming,
+ mock_create_route,
+ mock_request,
+ mock_response,
+ mock_user_api_key_dict,
+ ):
+ """Test that client Authorization header takes priority over server key."""
+ # Setup
+ mock_request.headers = {"authorization": "Bearer client-key-123"}
+ mock_router.get_credentials.return_value = "server-key-456"
+ mock_streaming.return_value = False
+ mock_endpoint_func = AsyncMock(return_value="test_response")
+ mock_create_route.return_value = mock_endpoint_func
+
+ # Act
+ await anthropic_proxy_route(
+ endpoint="v1/messages",
+ request=mock_request,
+ fastapi_response=mock_response,
+ user_api_key_dict=mock_user_api_key_dict,
+ )
+
+ # Assert
+ mock_create_route.assert_called_once()
+ call_kwargs = mock_create_route.call_args[1]
+
+ assert call_kwargs["custom_headers"] == {}
+ assert call_kwargs["_forward_headers"] is True
+
+ @pytest.mark.asyncio
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route")
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_streaming_request_fn")
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router")
+ async def test_client_x_api_key_header_priority(
+ self,
+ mock_router,
+ mock_streaming,
+ mock_create_route,
+ mock_request,
+ mock_response,
+ mock_user_api_key_dict,
+ ):
+ """Test that client x-api-key header takes priority over server key."""
+ # Setup
+ mock_request.headers = {"x-api-key": "client-x-api-key-123"}
+ mock_router.get_credentials.return_value = "server-key-456"
+ mock_streaming.return_value = False
+ mock_endpoint_func = AsyncMock(return_value="test_response")
+ mock_create_route.return_value = mock_endpoint_func
+
+ # Act
+ await anthropic_proxy_route(
+ endpoint="v1/messages",
+ request=mock_request,
+ fastapi_response=mock_response,
+ user_api_key_dict=mock_user_api_key_dict,
+ )
+
+ # Assert
+ mock_create_route.assert_called_once()
+ call_kwargs = mock_create_route.call_args[1]
+
+ assert call_kwargs["custom_headers"] == {}
+ assert call_kwargs["_forward_headers"] is True
+
+ @pytest.mark.asyncio
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route")
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_streaming_request_fn")
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router")
+ async def test_server_api_key_fallback(
+ self,
+ mock_router,
+ mock_streaming,
+ mock_create_route,
+ mock_request,
+ mock_response,
+ mock_user_api_key_dict,
+ ):
+ """Test that server API key is used when no client authentication is provided."""
+ # Setup
+ mock_request.headers = {} # No authentication headers
+ mock_router.get_credentials.return_value = "server-key-456"
+ mock_streaming.return_value = False
+ mock_endpoint_func = AsyncMock(return_value="test_response")
+ mock_create_route.return_value = mock_endpoint_func
+
+ # Act
+ await anthropic_proxy_route(
+ endpoint="v1/messages",
+ request=mock_request,
+ fastapi_response=mock_response,
+ user_api_key_dict=mock_user_api_key_dict,
+ )
+
+ # Assert
+ mock_create_route.assert_called_once()
+ call_kwargs = mock_create_route.call_args[1]
+
+ assert call_kwargs["custom_headers"] == {"x-api-key": "server-key-456"}
+ assert call_kwargs["_forward_headers"] is True
+
+ @pytest.mark.asyncio
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route")
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_streaming_request_fn")
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router")
+ async def test_no_authentication_available(
+ self,
+ mock_router,
+ mock_streaming,
+ mock_create_route,
+ mock_request,
+ mock_response,
+ mock_user_api_key_dict,
+ ):
+ """Test that no x-api-key header is added when no authentication is available."""
+ # Setup
+ mock_request.headers = {} # No authentication headers
+ mock_router.get_credentials.return_value = None # No server key
+ mock_streaming.return_value = False
+ mock_endpoint_func = AsyncMock(return_value="test_response")
+ mock_create_route.return_value = mock_endpoint_func
+
+ # Act
+ await anthropic_proxy_route(
+ endpoint="v1/messages",
+ request=mock_request,
+ fastapi_response=mock_response,
+ user_api_key_dict=mock_user_api_key_dict,
+ )
+
+ # Assert
+ mock_create_route.assert_called_once()
+ call_kwargs = mock_create_route.call_args[1]
+
+ assert call_kwargs["custom_headers"] == {}
+ assert call_kwargs["_forward_headers"] is True
+
+ @pytest.mark.asyncio
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route")
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_streaming_request_fn")
+ @patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router")
+ async def test_both_client_headers_present(
+ self,
+ mock_router,
+ mock_streaming,
+ mock_create_route,
+ mock_request,
+ mock_response,
+ mock_user_api_key_dict,
+ ):
+ """Test that no server key is added when client has both auth headers."""
+ # Setup
+ mock_request.headers = {
+ "authorization": "Bearer client-auth-key",
+ "x-api-key": "client-x-api-key"
+ }
+ mock_router.get_credentials.return_value = "server-key-456"
+ mock_streaming.return_value = False
+ mock_endpoint_func = AsyncMock(return_value="test_response")
+ mock_create_route.return_value = mock_endpoint_func
+
+ # Act
+ await anthropic_proxy_route(
+ endpoint="v1/messages",
+ request=mock_request,
+ fastapi_response=mock_response,
+ user_api_key_dict=mock_user_api_key_dict,
+ )
+
+ # Assert
+ mock_create_route.assert_called_once()
+ call_kwargs = mock_create_route.call_args[1]
+
+ assert call_kwargs["custom_headers"] == {}
+ assert call_kwargs["_forward_headers"] is True
\ No newline at end of file
diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
index 395f12ca111..ef733eaa887 100644
--- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
+++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py
@@ -322,3 +322,207 @@ class TestProxySettingEndpoints:
# Verify save_config was called exactly once
assert mock_proxy_config["save_call_count"]() == 1
+
+ def test_update_sso_settings_with_null_values_clears_env_vars(
+ self, mock_proxy_config, mock_auth, monkeypatch
+ ):
+ """Test that updating SSO settings with null values clears environment variables"""
+ monkeypatch.setenv("LITELLM_SALT_KEY", "test_salt_key")
+
+ # First, verify we have existing environment variables
+ initial_config = mock_proxy_config["config"]
+ assert "GOOGLE_CLIENT_ID" in initial_config["environment_variables"]
+ assert "MICROSOFT_CLIENT_ID" in initial_config["environment_variables"]
+
+ # Set some initial environment variables for runtime testing
+ monkeypatch.setenv("GOOGLE_CLIENT_ID", "test_existing_google_id")
+ monkeypatch.setenv("MICROSOFT_CLIENT_ID", "test_existing_microsoft_id")
+
+ # Send SSO settings with null values to clear them
+ clear_sso_settings = {
+ "google_client_id": None,
+ "google_client_secret": None,
+ "microsoft_client_id": None,
+ "microsoft_client_secret": None,
+ "microsoft_tenant": None,
+ "generic_client_id": None,
+ "generic_client_secret": None,
+ "generic_authorization_endpoint": None,
+ "generic_token_endpoint": None,
+ "generic_userinfo_endpoint": None,
+ "proxy_base_url": None,
+ "user_email": None,
+ "sso_provider": None,
+ }
+
+ response = client.patch("/update/sso_settings", json=clear_sso_settings)
+
+ assert response.status_code == 200
+ data = response.json()
+ assert data["status"] == "success"
+
+ # Verify that environment variables were cleared from config
+ updated_config = mock_proxy_config["config"]
+
+ # These should be removed from environment_variables
+ assert "GOOGLE_CLIENT_ID" not in updated_config["environment_variables"]
+ assert "GOOGLE_CLIENT_SECRET" not in updated_config["environment_variables"]
+ assert "MICROSOFT_CLIENT_ID" not in updated_config["environment_variables"]
+ assert "MICROSOFT_CLIENT_SECRET" not in updated_config["environment_variables"]
+ assert "MICROSOFT_TENANT" not in updated_config["environment_variables"]
+ assert "PROXY_BASE_URL" not in updated_config["environment_variables"]
+
+ # Verify that runtime environment variables were cleared
+ assert "GOOGLE_CLIENT_ID" not in os.environ
+ assert "MICROSOFT_CLIENT_ID" not in os.environ
+
+ # Verify user_email was cleared from general_settings
+ assert updated_config["general_settings"].get("proxy_admin_email") is None
+
+ # Verify save_config was called
+ assert mock_proxy_config["save_call_count"]() == 1
+
+ def test_update_sso_settings_with_empty_strings_clears_env_vars(
+ self, mock_proxy_config, mock_auth, monkeypatch
+ ):
+ """Test that updating SSO settings with empty strings also clears environment variables"""
+ monkeypatch.setenv("LITELLM_SALT_KEY", "test_salt_key")
+
+ # Set some initial environment variables for runtime testing
+ monkeypatch.setenv("GOOGLE_CLIENT_ID", "test_existing_google_id")
+ monkeypatch.setenv("MICROSOFT_CLIENT_SECRET", "test_existing_microsoft_secret")
+
+ # Send SSO settings with empty strings to clear them
+ clear_sso_settings = {
+ "google_client_id": "",
+ "google_client_secret": "",
+ "microsoft_client_secret": "",
+ "proxy_base_url": "",
+ "user_email": "",
+ }
+
+ response = client.patch("/update/sso_settings", json=clear_sso_settings)
+
+ assert response.status_code == 200
+ data = response.json()
+ assert data["status"] == "success"
+
+ # Verify that environment variables with empty strings were cleared from config
+ updated_config = mock_proxy_config["config"]
+ assert "GOOGLE_CLIENT_ID" not in updated_config["environment_variables"]
+ assert "GOOGLE_CLIENT_SECRET" not in updated_config["environment_variables"]
+ assert "MICROSOFT_CLIENT_SECRET" not in updated_config["environment_variables"]
+ assert "PROXY_BASE_URL" not in updated_config["environment_variables"]
+
+ # Verify that runtime environment variables were cleared
+ assert "GOOGLE_CLIENT_ID" not in os.environ
+ assert "MICROSOFT_CLIENT_SECRET" not in os.environ
+
+ # Verify user_email was cleared from general_settings
+ assert updated_config["general_settings"].get("proxy_admin_email") is None
+
+ # Verify save_config was called
+ assert mock_proxy_config["save_call_count"]() == 1
+
+ def test_update_sso_settings_mixed_null_and_valid_values(
+ self, mock_proxy_config, mock_auth, monkeypatch
+ ):
+ """Test updating SSO settings with mix of null and valid values"""
+ monkeypatch.setenv("LITELLM_SALT_KEY", "test_salt_key")
+
+ # Set some initial environment variables
+ monkeypatch.setenv("GOOGLE_CLIENT_ID", "old_google_id")
+ monkeypatch.setenv("MICROSOFT_CLIENT_ID", "old_microsoft_id")
+ monkeypatch.setenv("PROXY_BASE_URL", "old_proxy_url")
+
+ # Send mixed SSO settings - some null, some valid
+ mixed_sso_settings = {
+ "google_client_id": "new_google_client_id", # Valid value
+ "google_client_secret": None, # Null to clear
+ "microsoft_client_id": None, # Null to clear
+ "microsoft_client_secret": "new_microsoft_secret", # Valid value
+ "proxy_base_url": "https://newproxy.com", # Valid value
+ "user_email": None, # Null to clear
+ }
+
+ response = client.patch("/update/sso_settings", json=mixed_sso_settings)
+
+ assert response.status_code == 200
+ data = response.json()
+ assert data["status"] == "success"
+
+ # Verify the config was updated correctly
+ updated_config = mock_proxy_config["config"]
+
+ # Valid values should be set
+ assert (
+ updated_config["environment_variables"]["GOOGLE_CLIENT_ID"]
+ != "new_google_client_id"
+ ) # Encrypted
+ assert (
+ updated_config["environment_variables"]["MICROSOFT_CLIENT_SECRET"]
+ != "new_microsoft_secret"
+ ) # Encrypted
+ assert (
+ updated_config["environment_variables"]["PROXY_BASE_URL"]
+ != "https://newproxy.com"
+ ) # Encrypted
+
+ # Null values should be cleared
+ assert "GOOGLE_CLIENT_SECRET" not in updated_config["environment_variables"]
+ assert "MICROSOFT_CLIENT_ID" not in updated_config["environment_variables"]
+
+ # Verify runtime environment variables
+ assert os.environ.get("GOOGLE_CLIENT_ID") == "new_google_client_id"
+ assert os.environ.get("MICROSOFT_CLIENT_SECRET") == "new_microsoft_secret"
+ assert "GOOGLE_CLIENT_SECRET" not in os.environ
+ assert "MICROSOFT_CLIENT_ID" not in os.environ
+
+ # Verify user_email was cleared from general_settings
+ assert updated_config["general_settings"].get("proxy_admin_email") is None
+
+ # Verify save_config was called
+ assert mock_proxy_config["save_call_count"]() == 1
+
+ def test_update_sso_settings_ui_access_mode_handling(
+ self, mock_proxy_config, mock_auth, monkeypatch
+ ):
+ """Test that ui_access_mode is handled correctly in general_settings"""
+ monkeypatch.setenv("LITELLM_SALT_KEY", "test_salt_key")
+
+ # Test setting ui_access_mode
+ sso_settings_with_ui_mode = {
+ "ui_access_mode": "admin_only",
+ "user_email": "admin@test.com",
+ }
+
+ response = client.patch("/update/sso_settings", json=sso_settings_with_ui_mode)
+
+ assert response.status_code == 200
+ data = response.json()
+ assert data["status"] == "success"
+
+ # Verify ui_access_mode was set in general_settings (not environment_variables)
+ updated_config = mock_proxy_config["config"]
+ assert updated_config["general_settings"]["ui_access_mode"] == "admin_only"
+ assert (
+ updated_config["general_settings"]["proxy_admin_email"] == "admin@test.com"
+ )
+
+ # Verify ui_access_mode is NOT in environment_variables
+ assert "ui_access_mode" not in updated_config["environment_variables"]
+
+ # Test clearing ui_access_mode
+ clear_ui_mode = {"ui_access_mode": None, "user_email": None}
+
+ response = client.patch("/update/sso_settings", json=clear_ui_mode)
+
+ assert response.status_code == 200
+
+ # Verify ui_access_mode and user_email were cleared
+ updated_config = mock_proxy_config["config"]
+ assert updated_config["general_settings"].get("ui_access_mode") is None
+ assert updated_config["general_settings"].get("proxy_admin_email") is None
+
+ # Verify save_config was called twice (once for each update)
+ assert mock_proxy_config["save_call_count"]() == 2
diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py
index 631592f4c1f..954597dda25 100644
--- a/tests/test_litellm/test_main.py
+++ b/tests/test_litellm/test_main.py
@@ -5,6 +5,7 @@ import sys
import httpx
import pytest
import respx
+from fastapi.testclient import TestClient
sys.path.insert(
0, os.path.abspath("../..")
@@ -175,7 +176,7 @@ async def test_url_with_format_param(model, sync_mode, monkeypatch):
else:
response = await acompletion(**args, client=client)
print(response)
- except Exception:
+ except Exception as e:
pass
mock_client.assert_called()
@@ -268,6 +269,7 @@ def test_bedrock_latency_optimized_inference():
def test_custom_provider_with_extra_headers():
+ from litellm.llms.custom_httpx.http_handler import HTTPHandler
with patch.object(
litellm.llms.custom_httpx.http_handler.HTTPHandler, "post"
@@ -284,6 +286,7 @@ def test_custom_provider_with_extra_headers():
def test_custom_provider_with_extra_body():
+ from litellm.llms.custom_httpx.http_handler import HTTPHandler
with patch.object(
litellm.llms.custom_httpx.http_handler.HTTPHandler, "post"
@@ -1129,57 +1132,52 @@ def test_anthropic_disable_url_suffix_env_var():
# Test with environment variable disabled (default behavior)
with patch.dict(os.environ, {"ANTHROPIC_API_BASE": "https://api.example.com"}):
actual_api_base = None
-
+
with patch("litellm.main.anthropic_chat_completions") as mock_anthropic:
-
def capture_completion(**kwargs):
nonlocal actual_api_base
actual_api_base = kwargs.get("api_base")
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
return mock_response
-
+
mock_anthropic.completion = capture_completion
-
+
# This should append /v1/messages
completion(
model="anthropic/claude-3-sonnet",
messages=[{"role": "user", "content": "test"}],
- api_key="test-key",
+ api_key="test-key"
)
-
+
# Verify the api_base has /v1/messages appended
assert actual_api_base.endswith("/v1/messages")
assert actual_api_base == "https://api.example.com/v1/messages"
# Test with environment variable enabled
- with patch.dict(
- os.environ,
- {
- "ANTHROPIC_API_BASE": "https://api.example.com/custom/path",
- "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX": "true",
- },
- ):
+ with patch.dict(os.environ, {
+ "ANTHROPIC_API_BASE": "https://api.example.com/custom/path",
+ "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX": "true"
+ }):
actual_api_base = None
-
+
with patch("litellm.main.anthropic_chat_completions") as mock_anthropic:
-
def capture_completion(**kwargs):
nonlocal actual_api_base
actual_api_base = kwargs.get("api_base")
mock_response = MagicMock()
mock_response.choices = [MagicMock()]
return mock_response
-
+
mock_anthropic.completion = capture_completion
-
+
# This should NOT append /v1/messages
completion(
model="anthropic/claude-3-sonnet",
messages=[{"role": "user", "content": "test"}],
- api_key="test-key",
+ api_key="test-key"
)
-
+
# Verify the api_base does not have /v1/messages appended
assert actual_api_base == "https://api.example.com/custom/path"
assert not actual_api_base.endswith("/v1/messages")
@@ -1194,260 +1192,48 @@ def test_anthropic_text_disable_url_suffix_env_var():
# Test with environment variable disabled (default behavior)
with patch.dict(os.environ, {"ANTHROPIC_API_BASE": "https://api.example.com"}):
actual_api_base = None
-
+
with patch("litellm.main.base_llm_http_handler") as mock_handler:
-
def capture_completion(**kwargs):
nonlocal actual_api_base
actual_api_base = kwargs.get("api_base")
return MagicMock()
-
+
mock_handler.completion = capture_completion
-
+
# This should append /v1/complete
completion(
model="anthropic_text/claude-instant-1",
messages=[{"role": "user", "content": "test"}],
- api_key="test-key",
+ api_key="test-key"
)
-
+
# Verify the api_base has /v1/complete appended
assert actual_api_base.endswith("/v1/complete")
assert actual_api_base == "https://api.example.com/v1/complete"
# Test with environment variable enabled
- with patch.dict(
- os.environ,
- {
- "ANTHROPIC_API_BASE": "https://api.example.com/custom/complete",
- "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX": "true",
- },
- ):
+ with patch.dict(os.environ, {
+ "ANTHROPIC_API_BASE": "https://api.example.com/custom/complete",
+ "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX": "true"
+ }):
actual_api_base = None
-
+
with patch("litellm.main.base_llm_http_handler") as mock_handler:
-
def capture_completion(**kwargs):
nonlocal actual_api_base
actual_api_base = kwargs.get("api_base")
return MagicMock()
-
+
mock_handler.completion = capture_completion
-
+
# This should NOT append /v1/complete
completion(
model="anthropic_text/claude-instant-1",
messages=[{"role": "user", "content": "test"}],
- api_key="test-key",
+ api_key="test-key"
)
-
+
# Verify the api_base does not have /v1/complete appended
assert actual_api_base == "https://api.example.com/custom/complete"
assert not actual_api_base.endswith("/v1/complete")
-
-
-# Test header handling functionality
-def test_header_priority_and_merging():
- """Test that headers are properly merged with correct priority: request headers > extra_headers > global litellm.headers"""
- import litellm
- from unittest.mock import patch, MagicMock
-
- # Store original headers to restore later
- original_headers = litellm.headers
-
- try:
- # Set global headers
- litellm.headers = {
- "X-Global-Header": "global-value",
- "X-Shared-Header": "global",
- }
-
- captured_headers = {}
-
- with patch(
- "litellm.llms.custom_httpx.http_handler.HTTPHandler.post"
- ) as mock_post:
-
- def capture_headers(*args, **kwargs):
- captured_headers.update(kwargs.get("headers", {}))
- mock_response = MagicMock()
- mock_response.json.return_value = {
- "choices": [{"message": {"content": "test"}}],
- "usage": {"total_tokens": 10},
- }
- mock_response.status_code = 200
- return mock_response
-
- mock_post.side_effect = capture_headers
-
- # Test header merging
- try:
- litellm.completion(
- model="custom/test-model",
- messages=[{"role": "user", "content": "test"}],
- api_base="https://example.com/api",
- extra_headers={
- "X-Extra-Header": "extra-value",
- "X-Shared-Header": "extra",
- },
- headers={
- "X-Request-Header": "request-value",
- "X-Shared-Header": "request",
- },
- )
- except Exception:
- # Expected since we're mocking
- pass
-
- # Verify header priority: request > extra > global
- assert "X-Global-Header" in captured_headers
- assert "X-Extra-Header" in captured_headers
- assert "X-Request-Header" in captured_headers
- assert captured_headers["X-Global-Header"] == "global-value"
- assert captured_headers["X-Extra-Header"] == "extra-value"
- assert captured_headers["X-Request-Header"] == "request-value"
- # Request headers should override others
- assert captured_headers["X-Shared-Header"] == "request"
-
- finally:
- # Restore original headers
- litellm.headers = original_headers
-
-
-def test_anthropic_header_passing():
- """Test that custom headers are properly passed to Anthropic API calls"""
- from unittest.mock import patch, MagicMock
-
- captured_headers = {}
-
- with patch("litellm.llms.anthropic.chat.handler.HTTPHandler.post") as mock_post:
-
- def capture_headers(*args, **kwargs):
- captured_headers.update(kwargs.get("headers", {}))
- mock_response = MagicMock()
- mock_response.json.return_value = {
- "content": [{"text": "test response"}],
- "usage": {"input_tokens": 5, "output_tokens": 5},
- }
- mock_response.status_code = 200
- return mock_response
-
- mock_post.side_effect = capture_headers
-
- try:
- litellm.completion(
- model="claude-3-5-sonnet-latest",
- messages=[{"role": "user", "content": "Hello"}],
- extra_headers={
- "X-API-Gateway-Key": "gateway-123",
- "X-Tenant-ID": "tenant-456",
- },
- )
- except Exception:
- # Expected since we're mocking
- pass
-
- # Verify custom headers are included along with anthropic headers
- assert "X-API-Gateway-Key" in captured_headers
- assert "X-Tenant-ID" in captured_headers
- assert captured_headers["X-API-Gateway-Key"] == "gateway-123"
- assert captured_headers["X-Tenant-ID"] == "tenant-456"
- # Verify anthropic-specific headers are also present
- assert "x-api-key" in captured_headers
- assert "anthropic-version" in captured_headers
-
-
-def test_openai_header_passing():
- """Test that custom headers are properly passed to OpenAI API calls"""
- from unittest.mock import patch, MagicMock
-
- captured_extra_headers = {}
-
- with patch(
- "litellm.llms.openai.chat.gpt_transformation.OpenAIGPTConfig.transform_request"
- ) as mock_transform:
- with patch("litellm.completion_cost") as mock_cost:
- mock_cost.return_value = 0.0
- mock_transform.return_value = {"model": "gpt-4", "messages": []}
-
- with patch("openai.OpenAI") as mock_openai:
- mock_client = MagicMock()
- mock_openai.return_value = mock_client
- mock_response = MagicMock()
- mock_response.choices = [MagicMock()]
- mock_response.choices[0].message.content = "test response"
- mock_response.usage.total_tokens = 10
- mock_client.chat.completions.create.return_value = mock_response
-
- try:
- litellm.completion(
- model="gpt-4",
- messages=[{"role": "user", "content": "Hello"}],
- extra_headers={
- "X-Custom-Auth": "bearer-token",
- "X-Request-ID": "req-789",
- },
- )
- except Exception:
- # Expected since we're mocking
- pass
-
- # Verify extra_headers were passed to the OpenAI client
- call_args = mock_client.chat.completions.create.call_args
- if call_args:
- kwargs = call_args.kwargs
- assert "extra_headers" in kwargs
- extra_headers = kwargs["extra_headers"]
- assert "X-Custom-Auth" in extra_headers
- assert "X-Request-ID" in extra_headers
-
-
-def test_global_headers_functionality():
- """Test that global litellm.headers work correctly"""
- import litellm
- from unittest.mock import patch, MagicMock
-
- # Store original headers to restore later
- original_headers = litellm.headers
-
- try:
- # Set global headers
- litellm.headers = {"X-Company-ID": "acme-corp", "X-Environment": "production"}
-
- captured_headers = {}
-
- with patch(
- "litellm.llms.custom_httpx.http_handler.HTTPHandler.post"
- ) as mock_post:
-
- def capture_headers(*args, **kwargs):
- captured_headers.update(kwargs.get("headers", {}))
- mock_response = MagicMock()
- mock_response.json.return_value = {
- "choices": [{"message": {"content": "test"}}],
- "usage": {"total_tokens": 10},
- }
- mock_response.status_code = 200
- return mock_response
-
- mock_post.side_effect = capture_headers
-
- try:
- litellm.completion(
- model="custom/test-model",
- messages=[{"role": "user", "content": "test"}],
- api_base="https://example.com/api",
- )
- except Exception:
- # Expected since we're mocking
- pass
-
- # Verify global headers are included
- assert "X-Company-ID" in captured_headers
- assert "X-Environment" in captured_headers
- assert captured_headers["X-Company-ID"] == "acme-corp"
- assert captured_headers["X-Environment"] == "production"
-
- finally:
- # Restore original headers
- litellm.headers = original_headers
diff --git a/tests/test_litellm/test_shared_session_integration.py b/tests/test_litellm/test_shared_session_integration.py
new file mode 100644
index 00000000000..94ec12b4e56
--- /dev/null
+++ b/tests/test_litellm/test_shared_session_integration.py
@@ -0,0 +1,190 @@
+"""
+Integration tests for shared session functionality in main.py
+"""
+import os
+import sys
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+# Add the litellm directory to the path
+sys.path.insert(0, os.path.abspath("../../.."))
+
+import litellm
+
+
+class TestSharedSessionIntegration:
+ """Test cases for shared session integration in main.py"""
+
+ def test_acompletion_shared_session_parameter(self):
+ """Test that acompletion accepts shared_session parameter"""
+ import inspect
+
+ # Get the function signature
+ sig = inspect.signature(litellm.acompletion)
+ params = list(sig.parameters.keys())
+
+ # Verify shared_session parameter exists
+ assert 'shared_session' in params
+
+ # Verify the parameter type annotation
+ shared_session_param = sig.parameters['shared_session']
+ assert 'ClientSession' in str(shared_session_param.annotation)
+
+ # Verify default value is None
+ assert shared_session_param.default is None
+
+ def test_completion_shared_session_parameter(self):
+ """Test that completion accepts shared_session parameter"""
+ import inspect
+
+ # Get the function signature
+ sig = inspect.signature(litellm.completion)
+ params = list(sig.parameters.keys())
+
+ # Verify shared_session parameter exists
+ assert 'shared_session' in params
+
+ # Verify the parameter type annotation
+ shared_session_param = sig.parameters['shared_session']
+ assert 'ClientSession' in str(shared_session_param.annotation)
+
+ # Verify default value is None
+ assert shared_session_param.default is None
+
+ @pytest.mark.asyncio
+ async def test_acompletion_with_shared_session_mock(self):
+ """Test acompletion with mocked shared session (no actual API call)"""
+ import inspect
+
+ # Create a mock session
+ mock_session = MagicMock()
+ mock_session.closed = False
+
+ # Mock the completion function to avoid actual API calls
+ with patch('litellm.completion') as mock_completion:
+ mock_completion.return_value = {"choices": [{"message": {"content": "test"}}]}
+
+ # This should not raise an error even though we can't make actual API calls
+ try:
+ # We can't actually call acompletion without proper setup,
+ # but we can verify the parameter is accepted
+ sig = inspect.signature(litellm.acompletion)
+ assert 'shared_session' in sig.parameters
+ except Exception as e:
+ # Expected to fail due to missing API keys, but parameter should be valid
+ sig = inspect.signature(litellm.acompletion)
+ assert 'shared_session' in sig.parameters
+
+ def test_shared_session_passed_to_completion_kwargs(self):
+ """Test that shared_session is passed through completion_kwargs"""
+ # This test verifies that the shared_session parameter
+ # is properly included in the completion_kwargs dictionary
+
+ # We can't easily test the internal logic without mocking,
+ # but we can verify the parameter exists in the function signature
+ import inspect
+
+ sig = inspect.signature(litellm.acompletion)
+ shared_session_param = sig.parameters['shared_session']
+
+ # Verify the parameter is properly typed
+ assert 'ClientSession' in str(shared_session_param.annotation)
+ assert shared_session_param.default is None
+
+ def test_backward_compatibility(self):
+ """Test that existing code without shared_session still works"""
+ import inspect
+
+ # Verify that shared_session has a default value of None
+ sig = inspect.signature(litellm.acompletion)
+ shared_session_param = sig.parameters['shared_session']
+
+ # This ensures backward compatibility
+ assert shared_session_param.default is None
+
+ def test_type_annotations_consistency(self):
+ """Test that type annotations are consistent between acompletion and completion"""
+ import inspect
+
+ # Get signatures for both functions
+ acompletion_sig = inspect.signature(litellm.acompletion)
+ completion_sig = inspect.signature(litellm.completion)
+
+ # Get the shared_session parameters
+ acompletion_param = acompletion_sig.parameters['shared_session']
+ completion_param = completion_sig.parameters['shared_session']
+
+ # Verify they have the same type annotation
+ assert str(acompletion_param.annotation) == str(completion_param.annotation)
+
+ # Verify they have the same default value
+ assert acompletion_param.default == completion_param.default
+
+ def test_shared_session_parameter_position(self):
+ """Test that shared_session parameter is in the correct position"""
+ import inspect
+
+ sig = inspect.signature(litellm.acompletion)
+ params = list(sig.parameters.keys())
+
+ # Find the position of shared_session
+ shared_session_index = params.index('shared_session')
+
+ # It should be near the end, before **kwargs
+ assert shared_session_index > 0
+ assert shared_session_index < len(params) - 1 # Should be before **kwargs
+
+ # Verify it's after the main parameters
+ assert 'model' in params[:shared_session_index]
+ assert 'messages' in params[:shared_session_index]
+
+
+class TestSharedSessionUsage:
+ """Test cases demonstrating proper usage of shared sessions"""
+
+ def test_shared_session_usage_example(self):
+ """Test example usage pattern for shared sessions"""
+ # This test demonstrates the expected usage pattern
+ # without actually making API calls
+
+ import inspect
+
+ # Verify the function signature allows for the expected usage
+ sig = inspect.signature(litellm.acompletion)
+ params = sig.parameters
+
+ # Verify all expected parameters exist
+ expected_params = [
+ 'model', 'messages', 'shared_session'
+ ]
+
+ for param in expected_params:
+ assert param in params, f"Parameter {param} not found in acompletion signature"
+
+ # Verify shared_session is optional
+ assert params['shared_session'].default is None
+
+ def test_shared_session_with_other_parameters(self):
+ """Test that shared_session works with other parameters"""
+ import inspect
+
+ sig = inspect.signature(litellm.acompletion)
+ params = sig.parameters
+
+ # Verify shared_session doesn't conflict with other parameters
+ assert 'shared_session' in params
+ assert 'model' in params
+ assert 'messages' in params
+ assert 'timeout' in params
+
+ # Verify the parameter order makes sense
+ param_list = list(params.keys())
+ shared_session_index = param_list.index('shared_session')
+
+ # shared_session should be after the main parameters but before **kwargs
+ assert shared_session_index > param_list.index('model')
+ assert shared_session_index > param_list.index('messages')
+
+ # Should be before **kwargs (last parameter)
+ assert shared_session_index < len(param_list) - 1
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index 618a2902540..6e20e956068 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -1,7 +1,7 @@
import json
import os
import sys
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
import pytest
from jsonschema import validate
@@ -522,6 +522,12 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"input_cost_per_image": {"type": "number"},
"input_cost_per_image_above_128k_tokens": {"type": "number"},
"input_cost_per_token_above_200k_tokens": {"type": "number"},
+ "cache_read_input_token_cost_flex": {"type": "number"},
+ "cache_read_input_token_cost_priority": {"type": "number"},
+ "input_cost_per_token_flex": {"type": "number"},
+ "input_cost_per_token_priority": {"type": "number"},
+ "output_cost_per_token_flex": {"type": "number"},
+ "output_cost_per_token_priority": {"type": "number"},
"input_cost_per_pixel": {"type": "number"},
"input_cost_per_query": {"type": "number"},
"input_cost_per_request": {"type": "number"},
@@ -601,6 +607,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"supports_web_search": {"type": "boolean"},
"supports_url_context": {"type": "boolean"},
"supports_reasoning": {"type": "boolean"},
+ "supports_service_tier": {"type": "boolean"},
"tool_use_system_prompt_tokens": {"type": "number"},
"tpm": {"type": "number"},
"supported_endpoints": {
@@ -2428,8 +2435,8 @@ class TestGetValidModelsWithCLI:
{"id": "claude-3-sonnet", "object": "model"}
]
}
-
- with patch('requests.get', return_value=mock_response) as mock_get:
+
+ with patch.object(litellm.module_level_client, "get", return_value=mock_response) as mock_get:
# Test the exact pattern used in cli_token_usage.py
result = litellm.get_valid_models(
check_provider_endpoint=True,
@@ -2441,21 +2448,22 @@ class TestGetValidModelsWithCLI:
# Verify the function returns a list of model names
assert isinstance(result, list)
assert len(result) == 4
- assert "gpt-3.5-turbo" in result
- assert "gpt-4" in result
- assert "litellm_proxy/gemini/gemini-2.5-flash" in result
- assert "claude-3-sonnet" in result
+ # All models get prefixed with "litellm_proxy/" by the get_models method
+ assert "litellm_proxy/gpt-3.5-turbo" in result
+ assert "litellm_proxy/gpt-4" in result
+ # Note: This model already had the prefix, so it gets double-prefixed
+ assert "litellm_proxy/litellm_proxy/gemini/gemini-2.5-flash" in result
+ assert "litellm_proxy/claude-3-sonnet" in result
# Verify the HTTP request was made with correct parameters
mock_get.assert_called_once()
- call_args = mock_get.call_args
-
+ _, call_kwargs = mock_get.call_args
+
# Check that the request was made to the correct endpoint
- assert "http://localhost:4000/" in call_args[0][0]
- assert "/v1/models" in call_args[0][0]
-
+ assert call_kwargs["url"].startswith("http://localhost:4000/")
+ assert call_kwargs["url"].endswith("/v1/models")
+
# Check that the API key was included in headers
- assert "headers" in call_args.kwargs
- headers = call_args.kwargs["headers"]
- assert "Authorization" in headers
- assert "Bearer sk-test-cli-key-123" == headers["Authorization"]
+ assert "headers" in call_kwargs
+ headers = call_kwargs["headers"]
+ assert headers.get("Authorization") == "Bearer sk-test-cli-key-123"
diff --git a/tests/test_litellm/test_uuid_fallback.py b/tests/test_litellm/test_uuid_fallback.py
new file mode 100644
index 00000000000..e18bdabda8a
--- /dev/null
+++ b/tests/test_litellm/test_uuid_fallback.py
@@ -0,0 +1,11 @@
+import importlib
+
+
+def test_fastuuid_flag_exposed():
+ mod = importlib.import_module("litellm._uuid")
+ assert hasattr(mod, "FASTUUID_AVAILABLE")
+ assert hasattr(mod, "uuid4")
+ # Ensure uuid4 returns something that looks like a UUID string
+ val = str(mod.uuid4())
+ assert isinstance(val, str)
+ assert len(val) >= 8
diff --git a/ui/litellm-dashboard/public/assets/logos/oracle.svg b/ui/litellm-dashboard/public/assets/logos/oracle.svg
new file mode 100644
index 00000000000..0981dfcff28
--- /dev/null
+++ b/ui/litellm-dashboard/public/assets/logos/oracle.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/ui/litellm-dashboard/src/components/SSOModals.tsx b/ui/litellm-dashboard/src/components/SSOModals.tsx
index 4920cec03b3..56ec9393fa4 100644
--- a/ui/litellm-dashboard/src/components/SSOModals.tsx
+++ b/ui/litellm-dashboard/src/components/SSOModals.tsx
@@ -186,21 +186,21 @@ const SSOModals: React.FC = ({
}
try {
- // Clear all SSO settings by sending empty values
+ // Clear all SSO settings
const clearSettings = {
- google_client_id: '',
- google_client_secret: '',
- microsoft_client_id: '',
- microsoft_client_secret: '',
- microsoft_tenant: '',
- generic_client_id: '',
- generic_client_secret: '',
- generic_authorization_endpoint: '',
- generic_token_endpoint: '',
- generic_userinfo_endpoint: '',
- proxy_base_url: '',
- user_email: '',
- sso_provider: '',
+ google_client_id: null,
+ google_client_secret: null,
+ microsoft_client_id: null,
+ microsoft_client_secret: null,
+ microsoft_tenant: null,
+ generic_client_id: null,
+ generic_client_secret: null,
+ generic_authorization_endpoint: null,
+ generic_token_endpoint: null,
+ generic_userinfo_endpoint: null,
+ proxy_base_url: null,
+ user_email: null,
+ sso_provider: null,
};
await updateSSOSettings(accessToken, clearSettings);
diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx
index a76d7b64ffe..9af728832ec 100644
--- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx
@@ -459,6 +459,12 @@ const PROVIDER_CREDENTIAL_FIELDS: Record =
label: "API Key",
type: "password",
required: true
+ }],
+ [Providers.Oracle]: [{
+ key: "api_key",
+ label: "API Key",
+ type: "password",
+ required: true
}]
};
diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx
index 17aa637af89..744f8c117d6 100644
--- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx
+++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx
@@ -31,6 +31,7 @@ export enum Providers {
OpenAI_Text = "OpenAI Text Completion",
OpenAI_Text_Compatible = "OpenAI-Compatible Text Completion Models (Together AI, etc.)",
Openrouter = "Openrouter",
+ Oracle = "Oracle Cloud Infrastructure (OCI)",
Perplexity = "Perplexity",
Sambanova = "Sambanova",
TogetherAI = "TogetherAI",
@@ -67,6 +68,7 @@ export const provider_map: Record = {
Perplexity: "perplexity",
TogetherAI: "together_ai",
Openrouter: "openrouter",
+ Oracle: "oci",
FireworksAI: "fireworks_ai",
GradientAI: "gradient_ai",
Triton: "triton",
@@ -106,6 +108,7 @@ export const providerLogoMap: Record = {
[Providers.OpenAI_Text_Compatible]: `${asset_logos_folder}openai_small.svg`,
[Providers.OpenAI_Compatible]: `${asset_logos_folder}openai_small.svg`,
[Providers.Openrouter]: `${asset_logos_folder}openrouter.svg`,
+ [Providers.Oracle]: `${asset_logos_folder}oracle.svg`,
[Providers.Perplexity]: `${asset_logos_folder}perplexity-ai.svg`,
[Providers.Sambanova]: `${asset_logos_folder}sambanova.svg`,
[Providers.TogetherAI]: `${asset_logos_folder}togetherai.svg`,
@@ -166,6 +169,8 @@ export const getPlaceholder = (selectedProvider: string): string => {
return "azure_ai/command-r-plus";
} else if (selectedProvider == Providers.Azure) {
return "azure/my-deployment";
+ } else if (selectedProvider == Providers.Oracle) {
+ return "oci/xai.grok-4";
} else if (selectedProvider == Providers.Voyage) {
return "voyage/";
} else if (selectedProvider == Providers.JinaAI) {