Merge branch 'BerriAI:main' into main

.circleci/config.yml

@@ -1521,6 +1521,7 @@ jobs:
       - run: python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
       - run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
       - run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
+      - run: python ./tests/code_coverage_tests/check_fastuuid_usage.py
       - run: helm lint ./deploy/charts/litellm-helm
 
   db_migration_disable_update_check:

README.md

@@ -350,13 +350,21 @@ curl 'http://0.0.0.0:4000/key/generate' \
 
 [**Read the Docs**](https://docs.litellm.ai/docs/)
 
-## Contributing
+## Run in Developer mode
+### Services
+1. Setup .env file in root
+2. Run dependant services `docker-compose up db prometheus`
 
-Interested in contributing? Contributions to LiteLLM Python SDK, Proxy Server, and LLM integrations are both accepted and highly encouraged!
+### Backend
+1. (In root) create virtual environment `python -m venv .venv`
+2. Activate virtual environment `source .venv/bin/activate`
+3. Install dependencies `pip install -e ".[all]"`
+4. Start proxy backend `python litellm/proxy_cli.py`
 
-**Quick start:** `git clone` → `make install-dev` → `make format` → `make lint` → `make test-unit`
-
-See our comprehensive [Contributing Guide (CONTRIBUTING.md)](CONTRIBUTING.md) for detailed instructions.
+### Frontend
+1. Navigate to `ui/litellm-dashboard`
+2. Install dependencies `npm install`
+3. Run `npm run dev` to start the dashboard
 
 # Enterprise
 For companies that need better security, user management and professional support
@@ -434,18 +442,3 @@ All these checks must pass before your PR can be merged.
 </a>
 
 
-## Run in Developer mode
-### Services
-1. Setup .env file in root
-2. Run dependant services `docker-compose up db prometheus`
-
-### Backend
-1. (In root) create virtual environment `python -m venv .venv`
-2. Activate virtual environment `source .venv/bin/activate`
-3. Install dependencies `pip install -e ".[all]"`
-4. Start proxy backend `python3 /path/to/litellm/proxy_cli.py`
-
-### Frontend
-1. Navigate to `ui/litellm-dashboard`
-2. Install dependencies `npm install`
-3. Run `npm run dev` to start the dashboard

docs/my-website/docs/completion/shared_session.md

@@ -0,0 +1,213 @@
+# Shared Session Support
+
+## Overview
+
+LiteLLM now supports sharing `aiohttp.ClientSession` instances across multiple API calls to avoid creating unnecessary new sessions. This improves performance and resource utilization.
+
+## Usage
+
+### Basic Usage
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def main():
+    # Create a shared session
+    async with ClientSession() as shared_session:
+        # Use the same session for multiple calls
+        response1 = await acompletion(
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "Hello"}],
+            shared_session=shared_session
+        )
+        
+        response2 = await acompletion(
+            model="gpt-4o", 
+            messages=[{"role": "user", "content": "How are you?"}],
+            shared_session=shared_session
+        )
+        
+        # Both calls reuse the same session!
+
+asyncio.run(main())
+```
+
+### Without Shared Session (Default)
+
+```python
+import asyncio
+from litellm import acompletion
+
+async def main():
+    # Each call creates a new session
+    response1 = await acompletion(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "Hello"}]
+    )
+    
+    response2 = await acompletion(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "How are you?"}]
+    )
+    # Two separate sessions created
+
+asyncio.run(main())
+```
+
+## Benefits
+
+- **Performance**: Reuse HTTP connections across multiple calls
+- **Resource Efficiency**: Reduce memory and connection overhead
+- **Better Control**: Manage session lifecycle explicitly
+- **Debugging**: Easy to trace which calls use which sessions
+
+## Debug Logging
+
+Enable debug logging to see session reuse in action:
+
+```python
+import os
+import litellm
+
+# Enable debug logging
+os.environ['LITELLM_LOG'] = 'DEBUG'
+
+# You'll see logs like:
+# 🔄 SHARED SESSION: acompletion called with shared_session (ID: 12345)
+# ✅ SHARED SESSION: Reusing existing ClientSession (ID: 12345)
+```
+
+## Common Patterns
+
+### FastAPI Integration
+
+```python
+from fastapi import FastAPI
+import aiohttp
+import litellm
+
+app = FastAPI()
+
+@app.post("/chat")
+async def chat(messages: list[dict]):
+    # Create session per request
+    async with aiohttp.ClientSession() as session:
+        return await litellm.acompletion(
+            model="gpt-4o",
+            messages=messages,
+            shared_session=session
+        )
+```
+
+### Batch Processing
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def process_batch(messages_list):
+    async with ClientSession() as shared_session:
+        tasks = []
+        for messages in messages_list:
+            task = acompletion(
+                model="gpt-4o",
+                messages=messages,
+                shared_session=shared_session
+            )
+            tasks.append(task)
+        
+        # All tasks use the same session
+        results = await asyncio.gather(*tasks)
+        return results
+```
+
+### Custom Session Configuration
+
+```python
+import aiohttp
+import litellm
+
+# Create optimized session
+async with aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=180),
+    connector=aiohttp.TCPConnector(limit=300, limit_per_host=75)
+) as shared_session:
+    
+    response = await litellm.acompletion(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "Hello"}],
+        shared_session=shared_session
+    )
+```
+
+## Implementation Details
+
+The `shared_session` parameter is threaded through the entire LiteLLM call chain:
+
+1. **`acompletion()`** - Accepts `shared_session` parameter
+2. **`BaseLLMHTTPHandler`** - Passes session to HTTP client creation
+3. **`AsyncHTTPHandler`** - Uses existing session if provided
+4. **`LiteLLMAiohttpTransport`** - Reuses the session for HTTP requests
+
+## Backward Compatibility
+
+- **100% backward compatible** - Existing code works unchanged
+- **Optional parameter** - `shared_session=None` by default
+- **No breaking changes** - All existing functionality preserved
+
+## Testing
+
+Test the shared session functionality:
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def test_shared_session():
+    async with ClientSession() as session:
+        print(f"✅ Created session: {id(session)}")
+        
+        try:
+            response = await acompletion(
+                model="gpt-4o",
+                messages=[{"role": "user", "content": "Hello"}],
+                shared_session=session,
+                api_key="your-api-key"
+            )
+            print(f"Response: {response.choices[0].message.content}")
+        except Exception as e:
+            print(f"✅ Expected error: {type(e).__name__}")
+        
+        print("✅ Session control working!")
+
+asyncio.run(test_shared_session())
+```
+
+## Files Modified
+
+The shared session functionality was added to these files:
+
+- `litellm/main.py` - Added `shared_session` parameter to `acompletion()` and `completion()`
+- `litellm/llms/custom_httpx/http_handler.py` - Core session reuse logic
+- `litellm/llms/custom_httpx/llm_http_handler.py` - HTTP handler integration
+- `litellm/llms/openai/openai.py` - OpenAI provider integration
+- `litellm/llms/openai/common_utils.py` - OpenAI client creation
+- `litellm/llms/azure/chat/o_series_handler.py` - Azure O Series handler
+
+## Troubleshooting
+
+### Session Not Being Reused
+
+1. **Check debug logs**: Enable `LITELLM_LOG=DEBUG` to see session reuse messages
+2. **Verify session is not closed**: Ensure the session is still active when making calls
+3. **Check parameter passing**: Make sure `shared_session` is passed to all `acompletion()` calls
+
+### Performance Issues
+
+1. **Session configuration**: Tune `aiohttp.ClientSession` parameters for your use case
+2. **Connection limits**: Adjust `limit` and `limit_per_host` in `TCPConnector`
+3. **Timeout settings**: Configure appropriate timeouts for your environment

docs/my-website/docs/completion/usage.md

@@ -26,7 +26,6 @@ response = completion(
 
 print(response.usage)
 ```
-> **Note:** LiteLLM supports endpoint bridging—if a model does not natively support a requested endpoint, LiteLLM will automatically route the call to the correct supported endpoint (such as bridging `/chat/completions` to `/responses` or vice versa) based on the model's `mode`set in `model_prices_and_context_window`.
 
 ## Streaming Usage
 

docs/my-website/docs/enterprise.md

@@ -1,11 +1,6 @@
 import Image from '@theme/IdealImage';
 
 # Enterprise
-
-:::info
-✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
-:::
-
 For companies that need SSO, user management and professional support for LiteLLM Proxy
 
 :::info

docs/my-website/docs/fine_tuning.md

@@ -13,8 +13,6 @@ This is an Enterprise only endpoint [Get Started with Enterprise here](https://c
 | Feature | Supported | Notes | 
 |-------|-------|-------|
 | Supported Providers | OpenAI, Azure OpenAI, Vertex AI | - |
-
-#### ⚡️See an exhaustive list of supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
 | Cost Tracking | 🟡 | [Let us know if you need this](https://github.com/BerriAI/litellm/issues) |
 | Logging | ✅ | Works across all logging integrations |
 

docs/my-website/docs/getting_started.md

@@ -32,8 +32,7 @@ Next Steps 👉 [Call all supported models - e.g. Claude-2, Llama2-70b, etc.](./
 More details 👉
 
 - [Completion() function details](./completion/)
-- [Overview of supported models / providers on LiteLLM](./providers/)
-- [Search all models / providers](https://models.litellm.ai/)
+- [All supported models / providers on LiteLLM](./providers/)
 - [Build your own OpenAI proxy](https://github.com/BerriAI/liteLLM-proxy/tree/main)
 
 ## streaming

docs/my-website/docs/image_edits.md

@@ -18,9 +18,6 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
 | Supported LiteLLM Proxy Versions | 1.71.1+ | |
 | Supported LLM providers | **OpenAI** | Currently only `openai` is supported |
 
- #### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
-
-
 ## Usage
 
 ### LiteLLM Python SDK

docs/my-website/docs/image_generation.md

@@ -279,8 +279,6 @@ print(f"response: {response}")
 
 ## Supported Providers
 
-#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
-
 | Provider | Documentation Link |
 |----------|-------------------|
 | OpenAI | [OpenAI Image Generation →](./providers/openai) |

docs/my-website/docs/index.md

@@ -524,15 +524,6 @@ try:
 except OpenAIError as e:
     print(e)
 ```
-### See How LiteLLM Transforms Your Requests
-
-Want to understand how LiteLLM parses and normalizes your LLM API requests? Use the `/utils/transform_request` endpoint to see exactly how your request is transformed internally.
-
-You can try it out now directly on our Demo App!
-Go to the [LiteLLM API docs for transform_request](https://litellm-api.up.railway.app/#/llm%20utils/transform_request_utils_transform_request_post)
-
-LiteLLM will show you the normalized, provider-agnostic version of your request. This is useful for debugging, learning, and understanding how LiteLLM handles different providers and options.
-
 
 ### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
 LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, Helicone, Promptlayer, Traceloop, Slack

docs/my-website/docs/integrations/index.md

@@ -2,4 +2,17 @@
 
 This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK).
 
+## AI Agent Frameworks
+- **[Letta](./letta.md)** - Build stateful LLM agents with persistent memory using LiteLLM Proxy
+
+## Development Tools
+- **[OpenWebUI](../tutorials/openweb_ui.md)** - Self-hosted ChatGPT-style interface
+
+## Observability & Monitoring
+- **[Langfuse](../observability/langfuse_integration.md)** - LLM observability and analytics
+- **[Prometheus](../proxy/prometheus.md)** - Metrics collection and monitoring
+- **[PagerDuty](../proxy/pagerduty.md)** - Incident response and alerting
+- **[Datadog](../observability/datadog.md)**
+
+
 Click into each section to learn more about the integrations.

docs/my-website/docs/integrations/letta.md

@@ -0,0 +1,928 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Letta Integration
+
+[Letta](https://github.com/letta-ai/letta) (formerly MemGPT) is a framework for building stateful LLM agents with persistent memory. This guide shows how to integrate both LiteLLM SDK and LiteLLM Proxy with Letta to leverage multiple LLM providers while building memory-enabled agents.
+
+## What is Letta?
+
+Letta allows you to build LLM agents that can:
+- Maintain long-term memory across conversations
+- Use function calling for tool interactions
+- Handle large context windows efficiently
+- Persist agent state and memory
+
+## Prerequisites
+
+```bash
+pip install letta litellm
+```
+
+## Quick Start
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+### 1. Start LiteLLM Proxy
+
+First, create a configuration file for your LiteLLM proxy:
+
+```yaml
+# config.yaml
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+
+  - model_name: claude-3-sonnet
+    litellm_params:
+      model: anthropic/claude-3-sonnet-20240229
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: azure/gpt-35-turbo
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+      api_version: "2023-07-01-preview"
+```
+
+Start the proxy:
+
+```bash
+litellm --config config.yaml --port 4000
+```
+
+### 2. Configure Letta with LiteLLM Proxy
+
+Configure Letta to use your LiteLLM proxy endpoint:
+
+```python
+import letta
+from letta import create_client
+
+# Configure Letta to use LiteLLM proxy
+client = create_client()
+
+# Configure the LLM endpoint
+client.set_default_llm_config(
+    model="gpt-4",  # This should match a model from your LiteLLM config
+    model_endpoint_type="openai",
+    model_endpoint="http://localhost:4000",  # Your LiteLLM proxy URL
+    context_window=8192
+)
+
+# Configure embedding endpoint (optional)
+client.set_default_embedding_config(
+    embedding_endpoint_type="openai",
+    embedding_endpoint="http://localhost:4000",
+    embedding_model="text-embedding-ada-002"
+)
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+### 1. Configure LiteLLM SDK
+
+Set up your API keys and configure LiteLLM:
+
+```python
+import os
+import litellm
+
+# Set your API keys
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+# Optional: Configure default settings
+litellm.set_verbose = True  # For debugging
+```
+
+### 2. Create Custom LLM Wrapper for Letta
+
+Create a custom LLM wrapper that uses LiteLLM SDK:
+
+```python
+import letta
+from letta import create_client
+from letta.llm_api.llm_api_base import LLMConfig
+import litellm
+from typing import List, Dict, Any
+
+class LiteLLMWrapper:
+    def __init__(self, model: str):
+        self.model = model
+    
+    def chat_completions_create(self, messages: List[Dict], **kwargs):
+        # Use LiteLLM SDK for completion
+        response = litellm.completion(
+            model=self.model,
+            messages=messages,
+            **kwargs
+        )
+        return response
+
+# Configure Letta with custom LiteLLM wrapper
+client = create_client()
+
+# Set up LLM configuration using direct SDK integration
+llm_config = LLMConfig(
+    model="gpt-4",  # or "claude-3-sonnet", "azure/gpt-35-turbo", etc.
+    model_endpoint_type="openai",
+    context_window=8192
+)
+
+client.set_default_llm_config(llm_config)
+```
+
+</TabItem>
+</Tabs>
+
+### 3. Create and Use a Letta Agent
+
+<Tabs>
+<TabItem value="proxy" label="Using LiteLLM Proxy">
+
+```python
+import letta
+from letta import create_client
+
+# Create Letta client
+client = create_client()
+
+# Create a new agent
+agent_state = client.create_agent(
+    name="my-assistant",
+    system="You are a helpful assistant with persistent memory.",
+    llm_config=client.get_default_llm_config(),
+    embedding_config=client.get_default_embedding_config()
+)
+
+# Send a message to the agent
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="Hi! My name is Alice and I love reading science fiction books."
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+
+# Send another message - the agent will remember previous context
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="What did I tell you about my interests?"
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+```
+
+</TabItem>
+<TabItem value="sdk" label="Using LiteLLM SDK">
+
+```python
+import letta
+from letta import create_client
+import litellm
+import os
+
+# Set up environment variables
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Create Letta client with LiteLLM integration
+client = create_client()
+
+# Create a new agent
+agent_state = client.create_agent(
+    name="my-assistant",
+    system="You are a helpful assistant with persistent memory.",
+    llm_config=client.get_default_llm_config(),
+    embedding_config=client.get_default_embedding_config()
+)
+
+# Send a message to the agent
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="Hi! My name is Alice and I love reading science fiction books."
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+
+# Send another message - the agent will remember previous context
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="What did I tell you about my interests?"
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+```
+
+</TabItem>
+</Tabs>
+
+## Advanced Configuration
+
+### Using Different Models for Different Agents
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```python
+from letta import LLMConfig, EmbeddingConfig
+
+# Create different LLM configurations pointing to your proxy
+gpt4_config = LLMConfig(
+    model="gpt-4",
+    model_endpoint_type="openai",
+    model_endpoint="http://localhost:4000",
+    context_window=8192
+)
+
+claude_config = LLMConfig(
+    model="claude-3-sonnet",
+    model_endpoint_type="openai",  # Using OpenAI-compatible endpoint
+    model_endpoint="http://localhost:4000",
+    context_window=200000
+)
+
+# Create agents with different configurations
+research_agent = client.create_agent(
+    name="research-agent",
+    system="You are a research assistant specialized in analysis.",
+    llm_config=claude_config  # Use Claude for research tasks
+)
+
+creative_agent = client.create_agent(
+    name="creative-agent", 
+    system="You are a creative writing assistant.",
+    llm_config=gpt4_config  # Use GPT-4 for creative tasks
+)
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+import os
+import litellm
+from letta import LLMConfig, EmbeddingConfig
+
+# Set up API keys for different providers
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+# Create different LLM configurations for direct SDK usage
+gpt4_config = LLMConfig(
+    model="openai/gpt-4",  # Using LiteLLM model format
+    model_endpoint_type="openai",
+    context_window=8192
+)
+
+claude_config = LLMConfig(
+    model="anthropic/claude-3-sonnet-20240229",  # Using LiteLLM model format
+    model_endpoint_type="openai",
+    context_window=200000
+)
+
+# Create agents with different configurations
+research_agent = client.create_agent(
+    name="research-agent",
+    system="You are a research assistant specialized in analysis.",
+    llm_config=claude_config  # Use Claude for research tasks
+)
+
+creative_agent = client.create_agent(
+    name="creative-agent", 
+    system="You are a creative writing assistant.",
+    llm_config=gpt4_config  # Use GPT-4 for creative tasks
+)
+```
+
+</TabItem>
+</Tabs>
+
+### Function Calling with Tools
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```python
+# Define custom tools for your agent
+def search_web(query: str) -> str:
+    """Search the web for information"""
+    # Your web search implementation
+    return f"Search results for: {query}"
+
+def save_note(content: str) -> str:
+    """Save a note to persistent storage"""
+    # Your note saving implementation
+    return f"Note saved: {content}"
+
+# Create agent with tools (using proxy endpoint)
+agent_state = client.create_agent(
+    name="research-assistant",
+    system="You are a research assistant that can search the web and save notes.",
+    llm_config=client.get_default_llm_config(),
+    embedding_config=client.get_default_embedding_config(),
+    tools=[search_web, save_note]
+)
+
+# The agent can now use these tools
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="Search for recent developments in AI and save important findings."
+)
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+import litellm
+import os
+
+# Set up API keys
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Define custom tools for your agent
+def search_web(query: str) -> str:
+    """Search the web for information"""
+    # Your web search implementation
+    return f"Search results for: {query}"
+
+def save_note(content: str) -> str:
+    """Save a note to persistent storage"""
+    # Your note saving implementation
+    return f"Note saved: {content}"
+
+# Create agent with tools (using LiteLLM SDK directly)
+agent_state = client.create_agent(
+    name="research-assistant",
+    system="You are a research assistant that can search the web and save notes.",
+    llm_config=LLMConfig(
+        model="openai/gpt-4",  # Direct model specification
+        model_endpoint_type="openai",
+        context_window=8192
+    ),
+    embedding_config=client.get_default_embedding_config(),
+    tools=[search_web, save_note]
+)
+
+# The agent can now use these tools
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="Search for recent developments in AI and save important findings."
+)
+```
+
+</TabItem>
+</Tabs>
+
+## Authentication
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Authentication">
+
+If your LiteLLM proxy requires authentication:
+
+```python
+import os
+from letta import LLMConfig
+
+# Set up authenticated configuration
+llm_config = LLMConfig(
+    model="gpt-4",
+    model_endpoint_type="openai",
+    model_endpoint="http://localhost:4000",
+    model_wrapper="openai",
+    context_window=8192
+)
+
+# If using API keys with your proxy
+os.environ["OPENAI_API_KEY"] = "your-litellm-proxy-api-key"
+
+client = create_client()
+client.set_default_llm_config(llm_config)
+```
+
+For proxy with authentication enabled:
+
+```yaml
+# config.yaml with auth
+general_settings:
+  master_key: "your-master-key"
+
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+```
+
+```python
+# Configure Letta with authenticated proxy
+llm_config = LLMConfig(
+    model="gpt-4",
+    model_endpoint_type="openai",
+    model_endpoint="http://localhost:4000",
+    context_window=8192,
+    api_key="your-master-key"  # Proxy master key
+)
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Authentication">
+
+With LiteLLM SDK, set up your provider API keys directly:
+
+```python
+import os
+import litellm
+
+# Set up API keys for different providers
+os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-api-key" 
+os.environ["AZURE_API_KEY"] = "your-azure-api-key"
+os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com"
+os.environ["AZURE_API_VERSION"] = "2023-07-01-preview"
+
+# Optional: Configure default settings
+litellm.api_key = os.environ.get("OPENAI_API_KEY")  # Default key
+litellm.set_verbose = True  # For debugging
+
+# Use in Letta configuration
+from letta import LLMConfig
+
+llm_config = LLMConfig(
+    model="openai/gpt-4",  # Will use OPENAI_API_KEY automatically
+    model_endpoint_type="openai",
+    context_window=8192
+)
+
+# Or for Azure
+azure_config = LLMConfig(
+    model="azure/gpt-35-turbo", 
+    model_endpoint_type="openai",
+    context_window=4096
+)
+```
+
+</TabItem>
+</Tabs>
+
+## Load Balancing and Fallbacks
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Features">
+
+LiteLLM proxy's load balancing and fallback features work seamlessly with Letta:
+
+```yaml
+# config.yaml with fallbacks
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+    tpm: 40000
+    rpm: 500
+
+  - model_name: gpt-4  # Same model name for fallback
+    litellm_params:
+      model: azure/gpt-4
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+      api_version: "2023-07-01-preview"
+    tpm: 80000
+    rpm: 800
+
+router_settings:
+  routing_strategy: "usage-based-routing"
+  fallbacks: [{"gpt-4": ["azure/gpt-4"]}]
+```
+
+The proxy handles all routing, load balancing, and fallbacks transparently for Letta.
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Router">
+
+With LiteLLM SDK, you can set up routing and fallbacks programmatically:
+
+```python
+import litellm
+from litellm import Router
+
+# Configure router with multiple models
+router = Router(
+    model_list=[
+        {
+            "model_name": "gpt-4",
+            "litellm_params": {
+                "model": "openai/gpt-4",
+                "api_key": os.environ["OPENAI_API_KEY"]
+            },
+            "tpm": 40000,
+            "rpm": 500
+        },
+        {
+            "model_name": "gpt-4",  # Same name for fallback
+            "litellm_params": {
+                "model": "azure/gpt-4", 
+                "api_key": os.environ["AZURE_API_KEY"],
+                "api_base": os.environ["AZURE_API_BASE"],
+                "api_version": "2023-07-01-preview"
+            },
+            "tpm": 80000,
+            "rpm": 800
+        }
+    ],
+    fallbacks=[{"gpt-4": ["azure/gpt-4"]}],
+    routing_strategy="usage-based-routing"
+)
+
+# Create custom completion function for Letta
+def custom_completion(messages, model="gpt-4", **kwargs):
+    return router.completion(
+        model=model,
+        messages=messages,
+        **kwargs
+    )
+
+# Use with Letta by monkey-patching or custom wrapper
+litellm.completion = custom_completion
+```
+
+</TabItem>
+</Tabs>
+
+## Monitoring and Observability
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Monitoring">
+
+Enable logging to track your Letta agents' LLM usage through the proxy:
+
+```yaml
+# config.yaml with logging
+model_list:
+  # ... your models
+
+litellm_settings:
+  success_callback: ["langfuse"]  # or other observability tools
+  
+environment_variables:
+  LANGFUSE_PUBLIC_KEY: "your-key"
+  LANGFUSE_SECRET_KEY: "your-secret"
+```
+
+View metrics in the proxy dashboard:
+```bash
+# Start proxy with UI
+litellm --config config.yaml --port 4000 --detailed_debug
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Monitoring">
+
+Set up observability directly in your SDK integration:
+
+```python
+import litellm
+import os
+
+# Configure observability callbacks
+os.environ["LANGFUSE_PUBLIC_KEY"] = "your-key"
+os.environ["LANGFUSE_SECRET_KEY"] = "your-secret"
+
+# Set global callbacks
+litellm.success_callback = ["langfuse"]
+litellm.failure_callback = ["langfuse"]
+
+# Optional: Set up custom logging
+litellm.set_verbose = True
+
+# Create custom completion wrapper with logging
+def logged_completion(messages, model="gpt-4", **kwargs):
+    try:
+        response = litellm.completion(
+            model=model,
+            messages=messages,
+            **kwargs
+        )
+        # Custom logging logic here if needed
+        return response
+    except Exception as e:
+        # Custom error handling
+        print(f"LLM call failed: {e}")
+        raise
+
+# Use in Letta configuration
+litellm.completion = logged_completion
+```
+
+</TabItem>
+</Tabs>
+
+## Example: Multi-Agent System
+
+<Tabs>
+<TabItem value="proxy" label="Using LiteLLM Proxy">
+
+```python
+import letta
+from letta import create_client, LLMConfig
+
+client = create_client()
+
+# Create specialized agents using proxy endpoints
+agents = {}
+
+# Research agent using Claude for analysis
+agents['researcher'] = client.create_agent(
+    name="researcher",
+    system="You are a research specialist. Analyze information thoroughly.",
+    llm_config=LLMConfig(
+        model="claude-3-sonnet",
+        model_endpoint="http://localhost:4000",
+        model_endpoint_type="openai"
+    )
+)
+
+# Writer agent using GPT-4 for content creation
+agents['writer'] = client.create_agent(
+    name="writer",
+    system="You are a content writer. Create engaging, well-structured content.",
+    llm_config=LLMConfig(
+        model="gpt-4",
+        model_endpoint="http://localhost:4000", 
+        model_endpoint_type="openai"
+    )
+)
+
+# Coordinator workflow
+def research_and_write_workflow(topic: str):
+    # Research phase
+    research_response = client.user_message(
+        agent_id=agents['researcher'].id,
+        message=f"Research the topic: {topic}. Provide key insights and data."
+    )
+    
+    research_results = research_response.messages[-1].text
+    
+    # Writing phase
+    write_response = client.user_message(
+        agent_id=agents['writer'].id,
+        message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
+    )
+    
+    return write_response.messages[-1].text
+
+# Execute workflow
+article = research_and_write_workflow("The future of AI in healthcare")
+print(article)
+```
+
+</TabItem>
+<TabItem value="sdk" label="Using LiteLLM SDK">
+
+```python
+import letta
+from letta import create_client, LLMConfig
+import litellm
+import os
+
+# Set up environment
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+client = create_client()
+
+# Create specialized agents using direct SDK models
+agents = {}
+
+# Research agent using Claude for analysis
+agents['researcher'] = client.create_agent(
+    name="researcher",
+    system="You are a research specialist. Analyze information thoroughly.",
+    llm_config=LLMConfig(
+        model="anthropic/claude-3-sonnet-20240229",
+        model_endpoint_type="openai"
+    )
+)
+
+# Writer agent using GPT-4 for content creation
+agents['writer'] = client.create_agent(
+    name="writer",
+    system="You are a content writer. Create engaging, well-structured content.",
+    llm_config=LLMConfig(
+        model="openai/gpt-4",
+        model_endpoint_type="openai"
+    )
+)
+
+# Cost-conscious agent using GPT-3.5
+agents['reviewer'] = client.create_agent(
+    name="reviewer",
+    system="You are an editor. Review and improve content quality.",
+    llm_config=LLMConfig(
+        model="openai/gpt-3.5-turbo",
+        model_endpoint_type="openai"
+    )
+)
+
+# Enhanced workflow with multiple agents
+def enhanced_workflow(topic: str):
+    # Research phase
+    research_response = client.user_message(
+        agent_id=agents['researcher'].id,
+        message=f"Research the topic: {topic}. Provide key insights and data."
+    )
+    
+    research_results = research_response.messages[-1].text
+    
+    # Writing phase
+    write_response = client.user_message(
+        agent_id=agents['writer'].id,
+        message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
+    )
+    
+    draft_article = write_response.messages[-1].text
+    
+    # Review phase
+    review_response = client.user_message(
+        agent_id=agents['reviewer'].id,
+        message=f"Please review and improve this article:\n\n{draft_article}"
+    )
+    
+    return review_response.messages[-1].text
+
+# Execute enhanced workflow
+article = enhanced_workflow("The future of AI in healthcare")
+print(article)
+```
+
+</TabItem>
+</Tabs>
+
+## Best Practices
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Best Practices">
+
+1. **Model Selection**: Use appropriate models for different tasks:
+   - Claude for analysis and reasoning
+   - GPT-4 for creative tasks
+   - GPT-3.5-turbo for simple interactions
+
+2. **Proxy Configuration**:
+   - Set appropriate rate limits and timeouts
+   - Use fallbacks for reliability
+   - Enable authentication for production
+
+3. **Memory Management**: Letta handles memory automatically, but monitor usage with large contexts
+
+4. **Cost Optimization**: 
+   - Use the proxy's budgeting features to control costs
+   - Set up rate limiting per user/team
+   - Monitor token usage through proxy dashboard
+
+5. **Monitoring**: Enable observability to track agent performance and token usage
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Best Practices">
+
+1. **Model Selection**: Choose models based on task requirements:
+   - Use `openai/gpt-4` for complex reasoning
+   - Use `anthropic/claude-3-sonnet-20240229` for analysis
+   - Use `openai/gpt-3.5-turbo` for cost-effective simple tasks
+
+2. **Error Handling**: Implement robust error handling with retries:
+   ```python
+   import litellm
+   from litellm import completion
+   
+   # Set up retry logic
+   litellm.num_retries = 3
+   litellm.request_timeout = 60
+   
+   # Custom error handling
+   def safe_completion(**kwargs):
+       try:
+           return completion(**kwargs)
+       except Exception as e:
+           print(f"LLM call failed: {e}")
+           # Implement fallback logic
+           return completion(model="openai/gpt-3.5-turbo", **kwargs)
+   ```
+
+3. **Cost Management**:
+   - Use cheaper models for non-critical tasks
+   - Implement token counting and budgets
+   - Cache responses when appropriate
+
+4. **Performance**:
+   - Use async operations for concurrent requests
+   - Implement connection pooling
+   - Monitor response times
+
+5. **Security**:
+   - Store API keys securely (environment variables)
+   - Rotate keys regularly
+   - Implement rate limiting
+
+</TabItem>
+</Tabs>
+
+## Troubleshooting
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Issues">
+
+### Connection Issues
+```bash
+# Test your LiteLLM proxy
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-4",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+### Configuration Debugging
+```python
+# Enable verbose logging
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+# Test Letta configuration
+client = create_client()
+print(client.get_default_llm_config())
+```
+
+### Common Proxy Issues
+- **Port conflicts**: Make sure port 4000 isn't in use
+- **Model not found**: Verify model names match your config.yaml
+- **Authentication errors**: Check master key configuration
+- **Rate limiting**: Monitor proxy logs for rate limit hits
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Issues">
+
+### API Key Issues
+```python
+import os
+import litellm
+
+# Check if API keys are set
+print("OpenAI Key:", os.environ.get("OPENAI_API_KEY", "Not set"))
+print("Anthropic Key:", os.environ.get("ANTHROPIC_API_KEY", "Not set"))
+
+# Test direct LiteLLM call
+try:
+    response = litellm.completion(
+        model="openai/gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "Hello"}]
+    )
+    print("LiteLLM working:", response.choices[0].message.content)
+except Exception as e:
+    print("LiteLLM error:", e)
+```
+
+### Configuration Debugging
+```python
+# Enable verbose logging
+litellm.set_verbose = True
+
+# Test model availability
+models = ["openai/gpt-4", "anthropic/claude-3-sonnet-20240229"]
+for model in models:
+    try:
+        response = litellm.completion(
+            model=model,
+            messages=[{"role": "user", "content": "Test"}],
+            max_tokens=10
+        )
+        print(f"✓ {model} working")
+    except Exception as e:
+        print(f"✗ {model} failed: {e}")
+```
+
+### Common SDK Issues
+- **Import errors**: Ensure `pip install litellm letta` is run
+- **Model format**: Use `provider/model` format (e.g., `openai/gpt-4`)
+- **API key format**: Different providers have different key formats
+- **Rate limits**: Implement exponential backoff for retries
+
+</TabItem>
+</Tabs>
+
+## Resources
+
+- [Letta Documentation](https://docs.letta.ai/)
+- [LiteLLM Proxy Documentation](../proxy/quick_start.md)
+- [LiteLLM SDK Documentation](../completion/input.md)
+- [Function Calling Guide](../completion/function_call.md)
+- [Observability Setup](../observability/langfuse_integration.md)
+- [Router Configuration](../routing.md)

docs/my-website/docs/moderation.md

@@ -130,8 +130,6 @@ Here's the exact json output and type you can expect from all moderation calls:
 
 ## **Supported Providers**
 
-#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
-
 | Provider    |
 |-------------|
 | OpenAI      |  

docs/my-website/docs/observability/callbacks.md

@@ -5,15 +5,13 @@
 liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses.
 
 :::tip
-**New to LiteLLM Callbacks?**
-
-- For proxy/server logging and observability, see the [Proxy Logging Guide](https://docs.litellm.ai/docs/proxy/logging).
-- To write your own callback logic, see the [Custom Callbacks Guide](https://docs.litellm.ai/docs/observability/custom_callback).
+**New to LiteLLM Callbacks?** Check out our comprehensive [Callback Management Guide](./callback_management.md) to understand when to use different callback hooks like `async_log_success_event` vs `async_post_call_success_hook`.
 :::
 
+liteLLM supports:
 
-### Supported Callback Integrations
-
+- [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback)
+- [Callback Management Guide](./callback_management.md) - **Comprehensive guide for choosing the right hooks**
 - [Lunary](https://lunary.ai/docs)
 - [Langfuse](https://langfuse.com/docs)
 - [LangSmith](https://www.langchain.com/langsmith)
@@ -23,20 +21,9 @@ liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`,
 - [Sentry](https://docs.sentry.io/platforms/python/)
 - [PostHog](https://posthog.com/docs/libraries/python)
 - [Slack](https://slack.dev/bolt-python/concepts)
-- [Arize](https://docs.arize.com/)
-- [PromptLayer](https://docs.promptlayer.com/)
 
 This is **not** an extensive list. Please check the dropdown for all logging integrations.
 
-### Related Cookbooks
-Try out our cookbooks for code snippets and interactive demos:
-
-- [Langfuse Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Langfuse.ipynb)
-- [Lunary Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Lunary.ipynb)
-- [Arize Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Arize.ipynb)
-- [Proxy + Langfuse Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb)
-- [PromptLayer Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/LiteLLM_PromptLayer.ipynb)
-
 ### Quick Start
 
 ```python

docs/my-website/docs/observability/custom_callback.md

@@ -67,23 +67,6 @@ asyncio.run(completion())
 - `async_post_call_success_hook` - Access user data + modify responses
 - `async_pre_call_hook` - Modify requests before sending
 
-### Example: Modifying the Response in async_post_call_success_hook
-
-You can use `async_post_call_success_hook` to add custom headers or metadata to the response before it is returned to the client. For example:
-
-```python
-async def async_post_call_success_hook(data, user_api_key_dict, response):
-    # Add a custom header to the response
-    additional_headers = getattr(response, "_hidden_params", {}).get("additional_headers", {}) or {}
-    additional_headers["x-litellm-custom-header"] = "my-value"
-    if not hasattr(response, "_hidden_params"):
-        response._hidden_params = {}
-    response._hidden_params["additional_headers"] = additional_headers
-    return response
-```
-
-This allows you to inject custom metadata or headers into the response for downstream consumers. You can use this pattern to pass information to clients, proxies, or observability tools.
-
 ## Callback Functions
 If you just want to log on a specific event (e.g. on input) - you can use callback functions. 
 

docs/my-website/docs/providers/bedrock.md

@@ -2340,39 +2340,6 @@ response = completion(
 
 Make the bedrock completion call
 
----
-
-### Required AWS IAM Policy for AssumeRole
-
-To use `aws_role_name` (STS AssumeRole) with LiteLLM, your IAM user or role **must** have permission to call `sts:AssumeRole` on the target role. If you see an error like:
-
-```
-An error occurred (AccessDenied) when calling the AssumeRole operation: User: arn:aws:sts::...:assumed-role/litellm-ecs-task-role/... is not authorized to perform: sts:AssumeRole on resource: arn:aws:iam::...:role/Enterprise/BedrockCrossAccountConsumer
-```
-
-This means the IAM identity running LiteLLM does **not** have permission to assume the target role. You must update your IAM policy to allow this action.
-
-#### Example IAM Policy
-
-Replace `<TARGET_ROLE_ARN>` with the ARN of the role you want to assume (e.g., `arn:aws:iam::123456789012:role/Enterprise/BedrockCrossAccountConsumer`).
-
-```json
-{
-  "Version": "2012-10-17",
-  "Statement": [
-    {
-      "Effect": "Allow",
-      "Action": "sts:AssumeRole",
-      "Resource": "<TARGET_ROLE_ARN>"
-    }
-  ]
-}
-```
-
-**Note:** The target role itself must also trust the calling IAM identity (via its trust policy) for AssumeRole to succeed. See [AWS AssumeRole docs](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-api.html) for more details.
-
----
-
 <Tabs>
 <TabItem value="sdk" label="SDK">
 

docs/my-website/docs/providers/vertex.md

@@ -196,19 +196,7 @@ model_list:
       vertex_location: "us-central1"
       vertex_credentials: "/path/to/service_account.json" # [OPTIONAL] Do this OR `!gcloud auth application-default login` - run this to add vertex credentials to your env
 ```
-or
-```yaml
-model_list:
- - model_name: gemini-pro
-    litellm_params:
-      model: vertex_ai/gemini-1.5-pro
-      litellm_credential_name: vertex-global
-      vertex_project: project-name-here
-      vertex_location: global
-      base_model: gemini
-      model_info:
-        provider: Vertex
-```
+
 2. Start Proxy 
 
 ```
@@ -827,6 +815,77 @@ Use Vertex AI context caching is supported by calling provider api directly. (Un
 
 [**Go straight to provider**](../pass_through/vertex_ai.md#context-caching)
 
+#### 1. Create the Cache
+
+First, create the cache by sending a `POST` request to the `cachedContents` endpoint via the LiteLLM proxy.
+
+<Tabs>
+<TabItem value="proxy" label="PROXY">
+
+```bash
+curl http://0.0.0.0:4000/vertex_ai/v1/projects/{project_id}/locations/{location}/cachedContents \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "projects/{project_id}/locations/{location}/publishers/google/models/gemini-2.5-flash",
+    "displayName": "example_cache",
+    "contents": [{
+      "role": "user",
+      "parts": [{
+        "text": ".... a long book to be cached"
+      }]
+    }]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+#### 2. Get the Cache Name from the Response
+
+Vertex AI will return a response containing the `name` of the cached content. This name is the identifier for your cached data.
+
+```json
+{
+    "name": "projects/12341234/locations/{location}/cachedContents/123123123123123",
+    "model": "projects/{project_id}/locations/{location}/publishers/google/models/gemini-2.5-flash",
+    "createTime": "2025-09-23T19:13:50.674976Z",
+    "updateTime": "2025-09-23T19:13:50.674976Z",
+    "expireTime": "2025-09-23T20:13:50.655988Z",
+    "displayName": "example_cache",
+    "usageMetadata": {
+        "totalTokenCount": 1246,
+        "textCount": 5132
+    }
+}
+```
+
+#### 3. Use the Cached Content
+
+Use the `name` from the response as `cachedContent` or `cached_content` in subsequent API calls to reuse the cached information. This is passed in the body of your request to `/chat/completions`.
+
+<Tabs>
+<TabItem value="proxy" label="PROXY">
+
+```bash
+
+curl http://0.0.0.0:4000/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "cachedContent": "projects/545201925769/locations/us-central1/cachedContents/4511135542628319232",
+    "model": "gemini-2.5-flash",
+    "messages": [
+        {
+            "role": "user",
+            "content": "what is the book about?"
+        }
+    ]
+  }'
+```
+
+</TabItem>
+
 
 ## Pre-requisites
 * `pip install google-cloud-aiplatform` (pre-installed on proxy docker image)
@@ -2736,7 +2795,3 @@ Once that's done, when you deploy the new container in the Google Cloud Run serv
 
 
 s/o @[Darien Kindlund](https://www.linkedin.com/in/kindlund/) for this tutorial
-
-
-
-

docs/my-website/docs/proxy/caching.md

@@ -958,19 +958,6 @@ curl http://localhost:4000/v1/chat/completions \
 
 </Tabs>
 
-
-## Redis max_connections
-
-You can set the `max_connections` parameter in your `cache_params` for Redis. This is passed directly to the Redis client and controls the maximum number of simultaneous connections in the pool. If you see errors like `No connection available`, try increasing this value:
-
-```yaml
-litellm_settings:
-  cache: true
-  cache_params:
-    type: redis
-    max_connections: 100
-```
-
 ## Supported `cache_params` on proxy config.yaml
 
 ```yaml
@@ -979,7 +966,6 @@ cache_params:
   ttl: Optional[float]
   default_in_memory_ttl: Optional[float]
   default_in_redis_ttl: Optional[float]
-  max_connections: Optional[Int]
 
   # Type of cache (options: "local", "redis", "s3")
   type: s3

docs/my-website/docs/proxy/config_settings.md

@@ -50,7 +50,6 @@ litellm_settings:
     port: 6379  # The port number for the Redis cache. Required if type is "redis".
     password: "your_password"  # The password for the Redis cache. Required if type is "redis".
     namespace: "litellm.caching.caching" # namespace for redis cache
-    max_connections: 100  # [OPTIONAL] Set Maximum number of Redis connections. Passed directly to redis-py. 
   
     # Optional - Redis Cluster Settings
     redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}] 

docs/my-website/docs/proxy/custom_auth.md

@@ -21,6 +21,169 @@ async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
         raise Exception
 ```
 
+## UserAPIKeyAuth Fields Reference
+
+The `UserAPIKeyAuth` object supports the following fields for comprehensive auth configuration:
+
+### Core Authentication Fields
+```python
+UserAPIKeyAuth(
+    # Basic auth fields
+    api_key: Optional[str] = None,                    # The API key (will be hashed automatically)
+    token: Optional[str] = None,                      # Hashed token for internal use
+    key_name: Optional[str] = None,                   # Human-readable key name
+    key_alias: Optional[str] = None,                  # Key alias for identification
+    
+    # User identification
+    user_id: Optional[str] = None,                    # Unique user identifier
+    user_email: Optional[str] = None,                 # User email address
+    user_role: Optional[LitellmUserRoles] = None,     # User role (PROXY_ADMIN, INTERNAL_USER, etc.)
+    
+    # Team/Organization
+    team_id: Optional[str] = None,                    # Team identifier
+    team_alias: Optional[str] = None,                 # Team display name
+    org_id: Optional[str] = None,                     # Organization identifier
+)
+```
+
+### Budget and Spend Tracking
+```python
+UserAPIKeyAuth(
+    # User budgets
+    max_budget: Optional[float] = None,               # Maximum budget for the key
+    spend: float = 0.0,                              # Current spend amount
+    soft_budget: Optional[float] = None,              # Soft budget limit (warnings)
+    model_max_budget: Dict = {},                      # Per-model budget limits
+    model_spend: Dict = {},                           # Per-model spend tracking
+    
+    # Team budgets
+    team_max_budget: Optional[float] = None,          # Team's maximum budget
+    team_spend: Optional[float] = None,               # Team's current spend
+    team_member_spend: Optional[float] = None,        # This user's spend within the team
+    
+    # Budget timing
+    budget_duration: Optional[str] = None,            # Budget reset period
+    budget_reset_at: Optional[datetime] = None,       # When budget resets
+)
+```
+
+### Rate Limiting
+```python
+UserAPIKeyAuth(
+    # User limits
+    tpm_limit: Optional[int] = None,                  # Tokens per minute limit
+    rpm_limit: Optional[int] = None,                  # Requests per minute limit
+    user_tpm_limit: Optional[int] = None,             # User-specific TPM limit
+    user_rpm_limit: Optional[int] = None,             # User-specific RPM limit
+    
+    # Team limits
+    team_tpm_limit: Optional[int] = None,             # Team TPM limit
+    team_rpm_limit: Optional[int] = None,             # Team RPM limit
+    team_member_tpm_limit: Optional[int] = None,      # Per-member TPM limit
+    team_member_rpm_limit: Optional[int] = None,      # Per-member RPM limit
+    
+    # Per-model limits
+    rpm_limit_per_model: Optional[Dict[str, int]] = None,  # RPM limits by model
+    tpm_limit_per_model: Optional[Dict[str, int]] = None,  # TPM limits by model
+)
+```
+
+### End User Tracking
+```python
+UserAPIKeyAuth(
+    # End user identification and limits
+    end_user_id: Optional[str] = None,                # End user identifier
+    end_user_tpm_limit: Optional[int] = None,         # End user TPM limit
+    end_user_rpm_limit: Optional[int] = None,         # End user RPM limit
+    end_user_max_budget: Optional[float] = None,      # End user budget limit
+)
+```
+
+### Model and Route Access
+```python
+UserAPIKeyAuth(
+    # Model access control
+    models: List = [],                                # Allowed models list
+    team_models: List = [],                           # Team's allowed models
+    aliases: Dict = {},                               # Model aliases
+    
+    # Route permissions
+    allowed_routes: Optional[list] = [],              # Allowed API routes
+    allowed_cache_controls: Optional[list] = [],      # Cache control permissions
+    permissions: Dict = {},                           # General permissions
+)
+```
+
+### Advanced Configuration
+```python
+UserAPIKeyAuth(
+    # Request handling
+    max_parallel_requests: Optional[int] = None,      # Concurrent request limit
+    allowed_model_region: Optional[AllowedModelRegion] = None,  # Geographic restrictions
+    
+    # Expiration and status
+    expires: Optional[Union[str, datetime]] = None,   # Key expiration
+    blocked: Optional[bool] = None,                   # Whether key is blocked
+    
+    # Metadata and configuration
+    metadata: Dict = {},                              # Custom metadata
+    config: Dict = {},                               # Configuration settings
+    team_metadata: Optional[Dict] = None,             # Team metadata
+    
+    # Internal tracking
+    request_route: Optional[str] = None,              # Current request route
+    last_refreshed_at: Optional[float] = None,        # Cache refresh timestamp
+)
+```
+
+### Complete Example
+
+```python
+from datetime import datetime, timedelta
+from litellm.proxy._types import UserAPIKeyAuth, LitellmUserRoles
+
+async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth:
+    try:
+        # Example: Comprehensive auth configuration
+        if api_key.startswith("sk-admin-"):
+            return UserAPIKeyAuth(
+                api_key=api_key,
+                user_id="admin_user_123",
+                user_email="admin@company.com",
+                user_role=LitellmUserRoles.PROXY_ADMIN,
+                team_id="admin_team",
+                team_alias="Administrative Team",
+                max_budget=1000.0,
+                soft_budget=800.0,
+                tpm_limit=10000,
+                rpm_limit=100,
+                models=["gpt-4", "claude-3-sonnet", "gpt-3.5-turbo"],
+                allowed_routes=["/chat/completions", "/embeddings"],
+                expires=datetime.now() + timedelta(days=30),
+                metadata={"department": "engineering", "cost_center": "ai_ops"}
+            )
+        elif api_key.startswith("sk-team-"):
+            return UserAPIKeyAuth(
+                api_key=api_key,
+                user_id="team_user_456",
+                user_email="user@company.com",
+                user_role=LitellmUserRoles.INTERNAL_USER,
+                team_id="dev_team",
+                team_alias="Development Team",
+                max_budget=100.0,
+                tpm_limit=1000,
+                rpm_limit=20,
+                models=["gpt-3.5-turbo", "claude-3-haiku"],
+                team_member_tpm_limit=500,  # Limit within team
+                end_user_tpm_limit=100,     # Per end-user limit
+                metadata={"project": "chatbot_v2"}
+            )
+        else:
+            raise Exception("Invalid API key")
+    except Exception:
+        raise Exception("Authentication failed")
+```
+
 #### 2. Pass the filepath (relative to the config.yaml)
 
 Pass the filepath to the config.yaml 

docs/my-website/docs/proxy/custom_sso.md

@@ -1,7 +1,9 @@
 # ✨ Event Hooks for SSO Login
 
 :::info
-✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
+
+✨ This is an Enterprise only feature [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
+
 :::
 
 ## Overview

docs/my-website/docs/proxy/db_deadlocks.md

@@ -84,29 +84,3 @@ LiteLLM emits the following prometheus metrics to monitor the health/status of t
 | `litellm_in_memory_spend_update_queue_size`         | In-memory aggregate spend values for keys, users, teams, team members, etc.| In-Memory    |
 | `litellm_redis_spend_update_queue_size`             | Redis aggregate spend values for keys, users, teams, etc.                  | Redis        |
 
-
-## Troubleshooting: Redis Connection Errors
-
-You may see errors like:
-
-```
-LiteLLM Redis Caching: async async_increment() - Got exception from REDIS No connection available., Writing value=21
-LiteLLM Redis Caching: async set_cache_pipeline() - Got exception from REDIS No connection available., Writing value=None
-```
- 
-This means all available Redis connections are in use, and LiteLLM cannot obtain a new connection from the pool. This can happen under high load or with many concurrent proxy requests.
-
-**Solution:**
-
-- Increase the `max_connections` parameter in your Redis config section in `proxy_config.yaml` to allow more simultaneous connections. For example:
-
-```yaml
-litellm_settings:
-  cache: True
-  cache_params:
-    type: redis
-    max_connections: 100  # Increase as needed for your traffic
-```
-
-Adjust this value based on your expected concurrency and Redis server capacity.
-

docs/my-website/docs/proxy/dynamic_rate_limit.md

@@ -139,6 +139,8 @@ litellm_settings:
   priority_reservation: 
     "prod": 0.9  # 90% reserved for production (9 RPM)
     "dev": 0.1   # 10% reserved for development (1 RPM)
+  priority_reservation_settings:
+    default_priority: 0  # Weight (0%) assigned to keys without explicit priority metadata
 
 general_settings:
   master_key: sk-1234 # OR set `LITELLM_MASTER_KEY=".."` in your .env
@@ -152,6 +154,9 @@ general_settings:
 - **Value (float)**: Percentage of total TPM/RPM to reserve (0.0 to 1.0)
 - **Note**: Values should sum to 1.0 or less
 
+`priority_reservation_settings`: Object (Optional)
+- **default_priority (float)**: Weight/percentage (0.0 to 1.0) assigned to API keys that have no priority metadata set (defaults to 0.5)
+
 **Start Proxy**
 
 ```bash
@@ -180,6 +185,14 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \
 }'
 ```
 
+**Key Without Priority (uses default_priority weight):**
+```bash
+curl -X POST 'http://0.0.0.0:4000/key/generate' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{}'
+```
+
 **Expected Response for both:**
 ```json
 {
@@ -217,9 +230,10 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 
 With the configuration above:
 
-1. **Production keys** can make up to 9 requests per minute
-2. **Development keys** can make up to 1 request per minute  
-3. Production requests are never blocked by development usage
+1. **Production keys** can make up to 9 requests per minute (90% of 10 RPM)
+2. **Development keys** can make up to 1 request per minute (10% of 10 RPM)
+3. **Keys without explicit priority** get the default_priority weight (0 = 0%), which allocates 0 requests per minute (0% of 10 RPM)
+4. Named priorities in `priority_reservation` and keys with `default_priority` operate independently
 
 **Rate Limit Error Example:**
 ```json

docs/my-website/docs/proxy/guardrails/bedrock.md

@@ -4,10 +4,6 @@ import TabItem from '@theme/TabItem';
 
 # Bedrock Guardrails
 
-:::tip ⚡️
-If you haven't set up or authenticated your Bedrock provider yet, see the [Bedrock Provider Setup & Authentication Guide](../../providers/bedrock.md).
-:::
-
 LiteLLM supports Bedrock guardrails via the [Bedrock ApplyGuardrail API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ApplyGuardrail.html). 
 
 ## Quick Start

docs/my-website/docs/proxy/load_balancing.md

@@ -172,9 +172,6 @@ router_settings:
   redis_host: <your redis host>
   redis_password: <your redis password>
   redis_port: 1992
-  cache_params:
-    type: redis
-    max_connections: 100  # maximum Redis connections in the pool; tune based on expected concurrency/load
 ```
 
 ## Router settings on config - routing_strategy, model_group_alias

docs/my-website/docs/proxy/self_serve.md

@@ -227,7 +227,7 @@ export PROXY_LOGOUT_URL="https://www.google.com"
 <Image img={require('../../img/ui_logout.png')}  style={{ width: '400px', height: 'auto' }} />
 
 
-### Set default max budget for internal users 
+### Set max budget for internal users 
 
 Automatically apply budget per internal user when they sign up. By default the table will be checked every 10 minutes, for users to reset. To modify this, [see this](./users.md#reset-budgets)
 
@@ -239,10 +239,6 @@ litellm_settings:
 
 This sets a max budget of $10 USD for internal users when they sign up. 
 
-You can also manage these settings visually in the UI:
-
-<Image img={require('../../img/default_user_settings_admin_ui.png')}  style={{ width: '700px', height: 'auto' }} />
-
 This budget only applies to personal keys created by that user - seen under `Default Team` on the UI. 
 
 <Image img={require('../../img/max_budget_for_internal_users.png')}  style={{ width: '500px', height: 'auto' }} />

docs/my-website/docs/proxy/user_onboarding.md

@@ -0,0 +1,82 @@
+# User Onboarding Guide
+
+A step-by-step guide to help admins onboard users to your LiteLLM proxy instance and help users get started with their API key.
+
+---
+
+## For Administrators
+
+### Step 1: Create a User Account
+
+You can create a user account via the Admin UI or using the API.
+
+#### Admin UI
+- Go to the (`/ui` endpoint)
+- Navigate to the Internal Users section
+- Click "Add User" and fill in the required details
+
+#### API
+```bash
+curl -X POST http://localhost:4000/user/new \
+  -H "Authorization: Bearer <admin-key>" \
+  -H "Content-Type: application/json" \
+  -d '{"user_email": "user@example.com"}'
+```
+
+---
+
+### Step 2: Grant Access & Permissions
+
+- Assign the user to a team (optional)
+- Set budgets, rate limits, and allowed models as needed
+- Generate an API key for the user (via UI or API)
+
+#### **Generate API Key (API Example)**
+```bash
+curl -X POST http://localhost:4000/key/generate \
+  -H "Authorization: Bearer <admin-key>" \
+  -H "Content-Type: application/json" \
+  -d '{"user_id": "<user-id>", "max_budget": 100}'
+```
+
+---
+
+## For End Users
+
+### Step 3: Validate Your API Key
+
+Before making LLM calls, validate your key works by calling the `/v1/models` endpoint:
+
+```bash
+curl -X GET http://localhost:4000/v1/models \
+  -H "Authorization: Bearer <your-api-key>"
+```
+- If your key is valid, you'll get a list of available models.
+- If invalid, you'll get a 401 error.
+
+---
+
+### Step 4: Hello World - Make Your First LLM Call
+
+```bash
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Authorization: Bearer <your-api-key>" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "messages": [{"role": "user", "content": "Hello!"}]
+  }'
+```
+
+---
+
+## Troubleshooting
+- If you get a 401 error, check with your admin that your key is active and you have access to the requested model.
+- Use the `/v1/models` endpoint to quickly check if your key is valid without consuming LLM tokens.
+
+---
+
+## See Also
+- [Proxy Quick Start](./quick_start.md)
+- [User Management](./users.md)
+- [Key Management](./key_management.md)

docs/my-website/docs/proxy_api.md

@@ -27,7 +27,7 @@ Email us @ krrish@berri.ai
 ## Supported Models for LiteLLM Key
 These are the models that currently work with the "sk-litellm-.." keys.
 
-For a complete list of models/providers that you can call with LiteLLM, [check out our provider list](./providers/) or check out [models.litellm.ai](https://models.litellm.ai/)
+For a complete list of models/providers that you can call with LiteLLM, [check out our provider list](./providers/)
 
 * OpenAI models - [OpenAI docs](./providers/openai.md)
     * gpt-4

docs/my-website/docs/rerank.md

@@ -109,8 +109,6 @@ curl http://0.0.0.0:4000/rerank \
 
 ## **Supported Providers**
 
-#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
-
 | Provider    | Link to Usage      |
 |-------------|--------------------|
 | Cohere (v1 + v2 clients)      |   [Usage](#quick-start)                 |

docs/my-website/docs/response_api.md

@@ -3,11 +3,8 @@ import TabItem from '@theme/TabItem';
 
 # /responses [Beta]
 
-
 LiteLLM provides a BETA endpoint in the spec of [OpenAI's `/responses` API](https://platform.openai.com/docs/api-reference/responses)
 
-Requests to /chat/completions may be bridged here automatically when the provider lacks support for that endpoint. The model’s default `mode` determines how bridging works.(see `model_prices_and_context_window`) 
-
 | Feature | Supported | Notes |
 |---------|-----------|--------|
 | Cost Tracking | ✅ | Works with all supported models |
@@ -81,43 +78,6 @@ print(retrieved_response)
 # retrieved_response = await litellm.aget_responses(response_id=response_id)
 ```
 
-#### CANCEL a Response
-You can cancel an in-progress response (if supported by the provider):
-
-```python showLineNumbers title="Cancel Response by ID"
-import litellm
-
-# First, create a response
-response = litellm.responses(
-    model="openai/o1-pro",
-    input="Tell me a three sentence bedtime story about a unicorn.",
-    max_output_tokens=100
-)
-
-# Get the response ID
-response_id = response.id
-
-# Cancel the response by ID
-cancel_response = litellm.cancel_responses(
-    response_id=response_id
-)
-
-print(cancel_response)
-
-# For async usage
-# cancel_response = await litellm.acancel_responses(response_id=response_id)
-```
-
-
-**REST API:**
-```bash
-curl -X POST http://localhost:4000/v1/responses/response_id/cancel \
-    -H "Authorization: Bearer sk-1234"
-```
-
-This will attempt to cancel the in-progress response with the given ID.
-**Note:** Not all providers support response cancellation. If unsupported, an error will be raised.
-
 #### DELETE a Response
 ```python showLineNumbers title="Delete Response by ID"
 import litellm

docs/my-website/docs/sdk/headers.md

@@ -1,328 +0,0 @@
-# SDK Header Support
-
-LiteLLM SDK provides comprehensive support for passing additional headers with API requests. This is essential for enterprise environments using API gateways, service meshes, and multi-tenant architectures.
-
-## Overview
-
-Headers can be passed to LiteLLM in three ways, with the following priority order:
-1. **Request-specific headers** (highest priority)
-2. **extra_headers parameter**
-3. **Global litellm.headers** (lowest priority)
-
-When the same header key is specified in multiple places, the higher priority value will be used.
-
-## Usage Methods
-
-### 1. Global Headers (litellm.headers)
-
-Set headers that will be included in all API requests:
-
-```python
-import litellm
-
-# Set global headers for all requests
-litellm.headers = {
-    "X-API-Gateway-Key": "your-gateway-key",
-    "X-Company-ID": "acme-corp",
-    "X-Environment": "production"
-}
-
-# Now all completion calls will include these headers
-response = litellm.completion(
-    model="claude-3-5-sonnet-latest",
-    messages=[{"role": "user", "content": "Hello"}]
-)
-```
-
-### 2. Per-Request Headers (extra_headers)
-
-Pass headers for specific requests using the `extra_headers` parameter:
-
-```python
-import litellm
-
-response = litellm.completion(
-    model="gpt-4",
-    messages=[{"role": "user", "content": "Hello"}],
-    extra_headers={
-        "X-Request-ID": "req-12345",
-        "X-Tenant-ID": "tenant-abc",
-        "X-Custom-Auth": "bearer-token-xyz"
-    }
-)
-```
-
-### 3. Request Headers (headers parameter)
-
-Use the `headers` parameter for the highest priority header control:
-
-```python
-import litellm
-
-response = litellm.completion(
-    model="claude-3-5-sonnet-latest",
-    messages=[{"role": "user", "content": "Hello"}],
-    headers={
-        "X-Priority-Header": "high-priority-value",
-        "Authorization": "Bearer custom-token"
-    }
-)
-```
-
-### 4. Combining All Methods
-
-You can combine all three methods. Headers will be merged with the priority order:
-
-```python
-import litellm
-
-# Global headers (lowest priority)
-litellm.headers = {
-    "X-Company-ID": "acme-corp",
-    "X-Shared-Header": "global-value"
-}
-
-response = litellm.completion(
-    model="gpt-4",
-    messages=[{"role": "user", "content": "Hello"}],
-    extra_headers={
-        "X-Request-ID": "req-12345",
-        "X-Shared-Header": "extra-value"  # Overrides global
-    },
-    headers={
-        "X-Priority-Header": "important",
-        "X-Shared-Header": "request-value"  # Overrides both global and extra
-    }
-)
-
-# Final headers sent to API:
-# {
-#     "X-Company-ID": "acme-corp",           # From global
-#     "X-Request-ID": "req-12345",           # From extra_headers
-#     "X-Priority-Header": "important",      # From headers
-#     "X-Shared-Header": "request-value"     # From headers (highest priority)
-# }
-```
-
-## Enterprise Use Cases
-
-### API Gateway Integration (Apigee, Kong, AWS API Gateway)
-
-```python
-import litellm
-
-# Set up headers for API gateway routing and authentication
-litellm.headers = {
-    "X-API-Gateway-Key": "your-gateway-key",
-    "X-Route-Version": "v2"
-}
-
-# Per-tenant requests
-response = litellm.completion(
-    model="claude-3-5-sonnet-latest",
-    messages=[{"role": "user", "content": "Analyze this data"}],
-    extra_headers={
-        "X-Tenant-ID": "tenant-123",
-        "X-Department": "engineering"
-    }
-)
-```
-
-### Service Mesh (Istio, Linkerd)
-
-```python
-import litellm
-
-response = litellm.completion(
-    model="gpt-4",
-    messages=[{"role": "user", "content": "Hello"}],
-    extra_headers={
-        "X-Trace-ID": "trace-abc-123",
-        "X-Service-Name": "ai-service",
-        "X-Version": "1.2.3"
-    }
-)
-```
-
-### Multi-Tenant SaaS Applications
-
-```python
-import litellm
-
-def make_ai_request(user_id, tenant_id, content):
-    return litellm.completion(
-        model="claude-3-5-sonnet-latest",
-        messages=[{"role": "user", "content": content}],
-        extra_headers={
-            "X-User-ID": user_id,
-            "X-Tenant-ID": tenant_id,
-            "X-Request-Time": str(int(time.time()))
-        }
-    )
-
-# Usage
-response = make_ai_request("user-456", "tenant-org-1", "Help me write code")
-```
-
-### Request Tracing and Debugging
-
-```python
-import litellm
-import uuid
-
-def traced_completion(model, messages, **kwargs):
-    trace_id = str(uuid.uuid4())
-
-    return litellm.completion(
-        model=model,
-        messages=messages,
-        extra_headers={
-            "X-Trace-ID": trace_id,
-            "X-Debug-Mode": "true",
-            "X-Source-Service": "my-app"
-        },
-        **kwargs
-    )
-
-# Usage
-response = traced_completion(
-    model="gpt-4",
-    messages=[{"role": "user", "content": "Debug this issue"}]
-)
-```
-
-### Custom Authentication
-
-```python
-import litellm
-
-def get_custom_auth_token():
-    # Your custom authentication logic
-    return "custom-auth-token"
-
-response = litellm.completion(
-    model="claude-3-5-sonnet-latest",
-    messages=[{"role": "user", "content": "Hello"}],
-    headers={
-        "X-Custom-Auth": get_custom_auth_token(),
-        "X-Auth-Type": "custom"
-    }
-)
-```
-
-## Provider Support
-
-Headers are supported across all LiteLLM providers including:
-
-- **OpenAI** (GPT models)
-- **Anthropic** (Claude models)
-- **Cohere**
-- **Hugging Face**
-- **Custom providers**
-- **Azure OpenAI**
-- **AWS Bedrock**
-- **Google Vertex AI**
-
-Each provider will receive your custom headers along with their required authentication and API-specific headers.
-
-## Best Practices
-
-### 1. Use Meaningful Header Names
-```python
-# Good
-extra_headers = {
-    "X-Request-ID": "req-12345",
-    "X-Tenant-ID": "org-456"
-}
-
-# Avoid
-extra_headers = {
-    "custom1": "value1",
-    "h2": "value2"
-}
-```
-
-### 2. Include Tracing Information
-```python
-extra_headers = {
-    "X-Trace-ID": trace_id,
-    "X-Span-ID": span_id,
-    "X-Service-Name": "ai-service"
-}
-```
-
-### 3. Handle Sensitive Information Carefully
-```python
-# Don't log sensitive headers
-import os
-
-if os.getenv("ENVIRONMENT") != "production":
-    extra_headers["X-Debug-User"] = user_id
-```
-
-### 4. Use Environment-Specific Headers
-```python
-import os
-
-environment = os.getenv("ENVIRONMENT", "development")
-
-litellm.headers = {
-    "X-Environment": environment,
-    "X-Service-Version": os.getenv("SERVICE_VERSION", "unknown")
-}
-```
-
-## Troubleshooting
-
-### Headers Not Being Passed
-
-If your headers aren't reaching the API:
-
-1. **Check Header Names**: Ensure header names don't conflict with provider-specific headers
-2. **Verify Priority**: Remember that `headers` > `extra_headers` > `litellm.headers`
-3. **Test with Logging**: Enable verbose logging to see what headers are being sent
-
-```python
-import litellm
-
-# Enable debug logging
-litellm.set_verbose = True
-
-response = litellm.completion(
-    model="gpt-4",
-    messages=[{"role": "user", "content": "test"}],
-    extra_headers={"X-Debug": "test"}
-)
-```
-
-### Gateway or Proxy Issues
-
-If using API gateways or proxies:
-
-1. **Check Gateway Requirements**: Verify required headers for your gateway
-2. **Test Direct vs Gateway**: Compare direct API calls vs gateway calls
-3. **Validate Header Format**: Some gateways have header format requirements
-
-## Security Considerations
-
-1. **Don't Log Sensitive Headers**: Avoid logging authentication tokens or personal data
-2. **Use HTTPS**: Always use secure connections when passing sensitive headers
-3. **Validate Header Values**: Sanitize user-provided header values
-4. **Rotate Keys**: Regularly rotate any API keys passed in headers
-
-```python
-import litellm
-import re
-
-def safe_header_value(value):
-    # Remove potentially dangerous characters
-    return re.sub(r'[^\w\-.]', '', str(value))
-
-response = litellm.completion(
-    model="gpt-4",
-    messages=[{"role": "user", "content": "Hello"}],
-    extra_headers={
-        "X-User-ID": safe_header_value(user_id)
-    }
-)
-```

docs/my-website/sidebars.js

@@ -57,31 +57,32 @@ const sidebars = {
       type: "category",
       label: "Alerting & Monitoring",
       items: [
+        "proxy/prometheus",
         "proxy/alerting",
-        "proxy/pagerduty",
-        "proxy/prometheus"
-      ]
+        "proxy/pagerduty"
+      ].sort()
     },
     {
       type: "category",
       label: "[Beta] Prompt Management",
       items: [
-        "proxy/custom_prompt_management",
+        "proxy/prompt_management",
         "proxy/native_litellm_prompt",
-        "proxy/prompt_management"
-      ]
+        "proxy/custom_prompt_management"
+      ].sort()
     },
     {
       type: "category",
       label: "AI Tools (OpenWebUI, Claude Code, etc.)",
       items: [
-        "tutorials/claude_responses_api",
-        "tutorials/cost_tracking_coding",
-        "tutorials/github_copilot_integration",
+        "integrations/letta",
+        "tutorials/openweb_ui",
+        "tutorials/openai_codex",
         "tutorials/litellm_gemini_cli",
         "tutorials/litellm_qwen_code_cli",
-        "tutorials/openai_codex",
-        "tutorials/openweb_ui"
+        "tutorials/github_copilot_integration",
+        "tutorials/claude_responses_api",
+        "tutorials/cost_tracking_coding",
       ]
     },
 
@@ -111,115 +112,29 @@ const sidebars = {
           label: "Setup & Deployment",
           items: [
             "proxy/quick_start",
-            "proxy/cli",
-            "proxy/debugging",
+            "proxy/user_onboarding",
             "proxy/deploy",
-            "proxy/health",
-            "proxy/master_key_rotations",
-            "proxy/model_management",
             "proxy/prod",
+            "proxy/cli",
             "proxy/release_cycle",
+            "proxy/model_management",
+            "proxy/health",
+            "proxy/debugging",
+            "proxy/master_key_rotations",
           ],
         },
         "proxy/demo",
-        {
-          type: "category",
-          label: "Admin UI",
-          items: [
-            "proxy/admin_ui_sso",
-            "proxy/custom_root_ui",
-            "proxy/custom_sso",
-            "proxy/model_hub",
-            "proxy/public_teams",
-            "proxy/self_serve",
-            "proxy/ui",
-            "proxy/ui/bulk_edit_users",
-            "proxy/ui_credentials",
-            "tutorials/scim_litellm",
-            {
-              type: "category",
-              label: "UI Logs",
-              items: [
-                "proxy/ui_logs",
-                "proxy/ui_logs_sessions"
-              ]
-            }
-          ],
-        },
         {
           type: "category",
           label: "Architecture",
-          items: [
-            "proxy/architecture",
-            "proxy/control_plane_and_data_plane",
-            "proxy/db_deadlocks",
-            "proxy/db_info",
-            "proxy/image_handling",
-            "proxy/jwt_auth_arch",
-            "proxy/spend_logs_deletion",
-            "proxy/user_management_heirarchy",
-            "router_architecture"
-          ],
+          items: ["proxy/architecture", "proxy/control_plane_and_data_plane", "proxy/db_info", "proxy/db_deadlocks", "router_architecture", "proxy/user_management_heirarchy", "proxy/jwt_auth_arch", "proxy/image_handling", "proxy/spend_logs_deletion"],
         },
         {
           type: "link",
           label: "All Endpoints (Swagger)",
           href: "https://litellm-api.up.railway.app/",
         },
-  "proxy/enterprise",
-  "proxy/management_cli",
-        {
-          type: "category",
-          label: "Authentication",
-          items: [
-            "proxy/virtual_keys",
-            "proxy/token_auth",
-            "proxy/service_accounts",
-            "proxy/access_control",
-            "proxy/cli_sso",
-            "proxy/custom_auth",
-            "proxy/ip_address",
-            "proxy/email",
-            "proxy/multiple_admins",
-          ],
-        },
-        {
-          type: "category",
-          label: "Budgets + Rate Limits",
-          items: [
-            "proxy/customers",
-            "proxy/dynamic_rate_limit",
-            "proxy/rate_limit_tiers",
-            "proxy/team_budgets",
-            "proxy/temporary_budget_increase",
-            "proxy/users"
-          ],
-        },
-        "proxy/caching",
-        {
-          type: "category",
-          label: "Create Custom Plugins",
-          description: "Modify requests, responses, and more",
-          items: [
-            "proxy/call_hooks",
-            "proxy/rules",
-          ]
-        },
-        {
-          type: "link",
-          label: "Load Balancing, Routing, Fallbacks",
-          href: "https://docs.litellm.ai/docs/routing-load-balancing",
-        },
-        {
-          type: "category",
-          label: "Logging, Alerting, Metrics",
-          items: [
-            "proxy/dynamic_logging",
-            "proxy/logging",
-            "proxy/logging_spec",
-            "proxy/team_logging"
-          ],
-        },
+        "proxy/management_cli",
         {
           type: "category",
           label: "Making LLM Requests",
@@ -232,6 +147,19 @@ const sidebars = {
             "proxy/model_discovery",
           ],
         },
+        {
+          type: "category",
+          label: "Authentication",
+          items: [
+            "proxy/virtual_keys",
+            "proxy/token_auth",
+            "proxy/service_accounts",
+            "proxy/access_control",
+            "proxy/ip_address",
+            "proxy/email",
+            "proxy/custom_auth",
+          ],
+        },
         {
           type: "category",
           label: "Model Access",
@@ -240,6 +168,73 @@ const sidebars = {
             "proxy/team_model_add"
           ]
         },
+        {
+          type: "category",
+          label: "Spend Tracking",
+          items: ["proxy/cost_tracking", "proxy/custom_pricing", "proxy/billing",],
+        },
+        {
+          type: "category",
+          label: "Budgets + Rate Limits",
+          items: ["proxy/users", "proxy/temporary_budget_increase", "proxy/rate_limit_tiers", "proxy/team_budgets", "proxy/dynamic_rate_limit", "proxy/customers"],
+        },
+        {
+          type: "category",
+          label: "Enterprise Features",
+          items: [
+            "proxy/enterprise",
+            {
+              type: "category",
+              label: "Admin UI",
+              items: [
+                "proxy/ui",
+                "proxy/admin_ui_sso",
+                "proxy/custom_root_ui",
+                "proxy/model_hub",
+                "proxy/self_serve",
+                "proxy/public_teams",
+                "proxy/ui_credentials",
+                "proxy/ui/bulk_edit_users",
+                {
+                  type: "category",
+                  label: "UI Logs",
+                  items: [
+                    "proxy/ui_logs",
+                    "proxy/ui_logs_sessions"
+                  ]
+                }
+              ],
+            },
+            {
+              type: "category",
+              label: "SSO & Identity Management",
+              items: [
+                "proxy/cli_sso",
+                "proxy/admin_ui_sso",
+                "proxy/custom_sso",
+                "tutorials/scim_litellm",
+                "tutorials/msft_sso",
+                "proxy/multiple_admins",
+              ],
+            },
+          ],
+        },
+        {
+          type: "link",
+          label: "Load Balancing, Routing, Fallbacks",
+          href: "https://docs.litellm.ai/docs/routing-load-balancing",
+        },
+        {
+          type: "category",
+          label: "Logging, Alerting, Metrics",
+          items: [
+            "proxy/logging",
+            "proxy/logging_spec",
+            "proxy/team_logging",
+            "proxy/dynamic_logging"
+          ],
+        },
+
         {
           type: "category",
           label: "Secret Managers",
@@ -250,13 +245,14 @@ const sidebars = {
         },
         {
           type: "category",
-          label: "Spend Tracking",
+          label: "Create Custom Plugins",
+          description: "Modify requests, responses, and more",
           items: [
-            "proxy/billing",
-            "proxy/cost_tracking",
-            "proxy/custom_pricing"
-          ],
+            "proxy/call_hooks",
+            "proxy/rules",
+          ]
         },
+        "proxy/caching",
       ]
     },
     {
@@ -270,11 +266,13 @@ const sidebars = {
         slug: "/supported_endpoints",
       },
       items: [
+        "anthropic_unified",
+        "apply_guardrail",
         "assistants",
         {
           type: "category",
           label: "/audio",
-          items: [
+          "items": [
             "audio_transcription",
             "text_to_speech",
           ]
@@ -303,7 +301,6 @@ const sidebars = {
             "completion/http_handler_config",
           ],
         },
-        "text_completion",
         "embedding/supported_embedding",
         {
           type: "category",
@@ -321,14 +318,13 @@ const sidebars = {
             "proxy/managed_finetuning",
           ]
         },
-          "generateContent",
-          "apply_guardrail",
+        "generateContent",
         {
           type: "category",
           label: "/images",
           items: [
-            "image_edits",
             "image_generation",
+            "image_edits",
             "image_variations",
           ]
         },
@@ -339,23 +335,23 @@ const sidebars = {
           label: "Pass-through Endpoints (Anthropic SDK, etc.)",
           items: [
             "pass_through/intro",
-            "pass_through/anthropic_completion",
-            "pass_through/assembly_ai",
-            "pass_through/bedrock",
-            "pass_through/cohere",
+            "pass_through/vertex_ai",
             "pass_through/google_ai_studio",
-            "pass_through/langfuse",
+            "pass_through/cohere",
+            "pass_through/vllm",
             "pass_through/mistral",
             "pass_through/openai_passthrough",
-            "pass_through/vertex_ai",
-            "pass_through/vllm",
-            "proxy/pass_through"
-          ]
+            "pass_through/anthropic_completion",
+            "pass_through/bedrock",
+            "pass_through/assembly_ai",
+            "pass_through/langfuse",
+            "proxy/pass_through",
+          ],
         },
         "realtime",
         "rerank",
         "response_api",
-        "anthropic_unified",
+        "text_completion",
         {
           type: "category",
           label: "/vector_stores",
@@ -519,32 +515,33 @@ const sidebars = {
       type: "category",
       label: "Guides",
       items: [
-        "completion/audio",
-        "completion/batching",
-        "completion/computer_use",
-        "completion/document_understanding",
-        "completion/drop_params",
-        "completion/function_call",
-        "completion/image_generation_chat",
-        "completion/json_mode",
-        "completion/knowledgebase",
-        "completion/message_trimming",
-        "completion/model_alias",
-        "completion/mock_requests",
-        "completion/predict_outputs",
-        "completion/prefix",
-        "completion/prompt_caching",
-        "completion/prompt_formatting",
-        "completion/reliable_completions",
-        "completion/stream",
-        "completion/provider_specific_params",
-        "completion/vision",
-        "completion/web_search",
         "exception_mapping",
+        "completion/provider_specific_params",
         "guides/finetuned_models",
         "guides/security_settings",
+        "completion/audio",
+        "completion/image_generation_chat",
+        "completion/web_search",
+        "completion/document_understanding",
+        "completion/vision",
+        "completion/json_mode",
+        "reasoning_content",
+        "completion/computer_use",
+        "completion/prompt_caching",
+        "completion/predict_outputs",
+        "completion/knowledgebase",
+        "completion/prefix",
+        "completion/drop_params",
+        "completion/prompt_formatting",
+        "completion/stream",
+        "completion/message_trimming",
+        "completion/function_call",
+        "completion/model_alias",
+        "completion/batching",
+        "completion/mock_requests",
+        "completion/reliable_completions",
         "proxy/veo_video_generation",
-        "reasoning_content"
+
       ]
     },
 
@@ -557,37 +554,26 @@ const sidebars = {
         description: "Learn how to load balance, route, and set fallbacks for your LLM requests",
         slug: "/routing-load-balancing",
       },
-      items: [
-        "routing",
-        "scheduler",
-        "proxy/auto_routing",
-        "proxy/load_balancing",
-        "proxy/provider_budget_routing",
-        "proxy/reliability",
-        "proxy/tag_routing",
-        "proxy/timeout",
-        "wildcard_routing"
-      ],
+      items: ["routing", "scheduler", "proxy/load_balancing", "proxy/reliability", "proxy/timeout", "proxy/auto_routing", "proxy/tag_routing", "proxy/provider_budget_routing", "wildcard_routing"],
     },
     {
       type: "category",
       label: "LiteLLM Python SDK",
       items: [
         "set_keys",
-        "budget_manager",
-        "caching/all_caches",
         "completion/token_usage",
         "sdk/headers",
         "sdk_custom_pricing",
         "embedding/async_embedding",
         "embedding/moderation",
+        "budget_manager",
+        "caching/all_caches",
         "migration",
-        "sdk_custom_pricing",
         {
           type: "category",
           label: "LangChain, LlamaIndex, Instructor Integration",
           items: ["langchain/langchain", "tutorials/instructor"],
-        }
+        },
       ],
     },
 

examples/sdk_headers_example.py

@@ -1,195 +0,0 @@
-#!/usr/bin/env python3
-"""
-Example demonstrating LiteLLM SDK header support for enterprise environments.
-
-This example shows how to use additional headers with API gateways, service meshes,
-and multi-tenant architectures.
-"""
-
-import litellm
-import os
-from typing import Dict, Any
-
-def example_global_headers():
-    """Example: Set global headers for all requests"""
-    print("=== Global Headers Example ===")
-
-    # Set global headers that will be included in all API requests
-    litellm.headers = {
-        "X-API-Gateway-Key": "your-gateway-key-here",
-        "X-Company-ID": "acme-corp",
-        "X-Environment": "production"
-    }
-
-    print("Global headers set:", litellm.headers)
-
-    # These headers will now be included in all completion calls
-    # (Note: This example doesn't actually make API calls)
-    print("Global headers will be included in all subsequent completion() calls")
-
-
-def example_per_request_headers():
-    """Example: Using extra_headers for specific requests"""
-    print("\n=== Per-Request Headers Example ===")
-
-    headers_to_send = {
-        "X-Request-ID": "req-12345",
-        "X-Tenant-ID": "tenant-abc",
-        "X-Custom-Auth": "bearer-token-xyz"
-    }
-
-    print("Per-request headers:", headers_to_send)
-
-    # Example of how you would use extra_headers in a real call
-    # response = litellm.completion(
-    #     model="claude-3-5-sonnet-latest",
-    #     messages=[{"role": "user", "content": "Hello"}],
-    #     extra_headers=headers_to_send
-    # )
-
-
-def example_header_priority():
-    """Example: Demonstrating header priority and merging"""
-    print("\n=== Header Priority Example ===")
-
-    # Set global headers
-    litellm.headers = {
-        "X-Company-ID": "acme-corp",
-        "X-Shared-Header": "global-value"
-    }
-
-    # Headers that would be sent in a request
-    extra_headers = {
-        "X-Request-ID": "req-12345",
-        "X-Shared-Header": "extra-value"  # Overrides global
-    }
-
-    request_headers = {
-        "X-Priority-Header": "important",
-        "X-Shared-Header": "request-value"  # Overrides both global and extra
-    }
-
-    print("Global headers:", litellm.headers)
-    print("Extra headers:", extra_headers)
-    print("Request headers:", request_headers)
-    print("\nFinal headers would be:")
-    print("  X-Company-ID: acme-corp (from global)")
-    print("  X-Request-ID: req-12345 (from extra)")
-    print("  X-Priority-Header: important (from request)")
-    print("  X-Shared-Header: request-value (request wins - highest priority)")
-
-
-def example_enterprise_api_gateway():
-    """Example: Enterprise API Gateway scenario"""
-    print("\n=== Enterprise API Gateway Example ===")
-
-    # Simulate enterprise environment with Apigee or similar
-    gateway_config = {
-        "X-API-Gateway-Key": os.getenv("API_GATEWAY_KEY", "demo-key"),
-        "X-Route-Version": "v2",
-        "X-Rate-Limit-Group": "premium"
-    }
-
-    # Set gateway headers globally
-    litellm.headers = gateway_config
-    print("Gateway headers configured:", gateway_config)
-
-    # Function to make tenant-specific requests
-    def make_tenant_request(tenant_id: str, user_id: str, content: str) -> Dict[str, Any]:
-        """Make an AI request with tenant-specific headers"""
-
-        tenant_headers = {
-            "X-Tenant-ID": tenant_id,
-            "X-User-ID": user_id,
-            "X-Request-Time": "2024-01-01T00:00:00Z",
-            "X-Service-Name": "ai-assistant"
-        }
-
-        print(f"Making request for tenant {tenant_id}, user {user_id}")
-        print("Tenant-specific headers:", tenant_headers)
-
-        # In a real scenario, this would make the actual API call:
-        # return litellm.completion(
-        #     model="claude-3-5-sonnet-latest",
-        #     messages=[{"role": "user", "content": content}],
-        #     extra_headers=tenant_headers
-        # )
-
-        # For demo purposes, return mock data
-        return {"mock": "response", "headers_used": {**gateway_config, **tenant_headers}}
-
-    # Example usage
-    result = make_tenant_request("tenant-123", "user-456", "Analyze this data")
-    print("Response:", result)
-
-
-def example_service_mesh():
-    """Example: Service mesh integration (Istio, Linkerd)"""
-    print("\n=== Service Mesh Example ===")
-
-    service_mesh_headers = {
-        "X-Trace-ID": "trace-abc-123",
-        "X-Span-ID": "span-def-456",
-        "X-Service-Name": "ai-service",
-        "X-Version": "1.2.3",
-        "X-Cluster": "prod-us-west-2"
-    }
-
-    print("Service mesh headers:", service_mesh_headers)
-
-    # Example of using these headers for distributed tracing
-    # response = litellm.completion(
-    #     model="gpt-4",
-    #     messages=[{"role": "user", "content": "Hello"}],
-    #     extra_headers=service_mesh_headers
-    # )
-
-
-def example_debugging_and_monitoring():
-    """Example: Request debugging and monitoring"""
-    print("\n=== Debugging and Monitoring Example ===")
-
-    import uuid
-    import time
-
-    # Generate unique identifiers for request tracking
-    trace_id = str(uuid.uuid4())
-    request_id = f"req-{int(time.time())}"
-
-    debug_headers = {
-        "X-Trace-ID": trace_id,
-        "X-Request-ID": request_id,
-        "X-Debug-Mode": "true",
-        "X-Source-Service": "customer-support-bot",
-        "X-Request-Priority": "high"
-    }
-
-    print("Debug headers:", debug_headers)
-    print(f"Trace ID: {trace_id}")
-    print(f"Request ID: {request_id}")
-
-    # These headers help with:
-    # 1. Distributed tracing across services
-    # 2. Request correlation in logs
-    # 3. Debug mode enablement
-    # 4. Priority-based routing
-
-
-if __name__ == "__main__":
-    print("LiteLLM SDK Header Support Examples")
-    print("=" * 50)
-
-    example_global_headers()
-    example_per_request_headers()
-    example_header_priority()
-    example_enterprise_api_gateway()
-    example_service_mesh()
-    example_debugging_and_monitoring()
-
-    print("\n" + "=" * 50)
-    print("All examples completed!")
-    print("\nTo use in your application:")
-    print("1. Set litellm.headers for global headers")
-    print("2. Use extra_headers parameter for request-specific headers")
-    print("3. Use headers parameter for highest priority headers")
-    print("4. Headers are merged with priority: headers > extra_headers > litellm.headers")

litellm/__init__.py

@@ -89,6 +89,7 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
     LiteLLM_UpperboundKeyGenerateParams,
 )
 from litellm.types.utils import StandardKeyGenerationConfig, LlmProviders
+from litellm.types.utils import PriorityReservationSettings
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
 import httpx
@@ -373,6 +374,7 @@ public_model_groups: Optional[List[str]] = None
 public_model_groups_links: Dict[str, str] = {}
 #### REQUEST PRIORITIZATION ######
 priority_reservation: Optional[Dict[str, float]] = None
+priority_reservation_settings: "PriorityReservationSettings" = PriorityReservationSettings()
 
 
 ######## Networking Settings ########

litellm/_uuid.py

@@ -0,0 +1,23 @@
+"""
+Internal unified UUID helper.
+
+Tries to use fastuuid (performance) and falls back to stdlib uuid if unavailable.
+"""
+
+FASTUUID_AVAILABLE = False
+
+try:
+    import fastuuid as _uuid  # type: ignore
+
+    FASTUUID_AVAILABLE = True
+except Exception:  # pragma: no cover - fallback path
+    import uuid as _uuid  # type: ignore
+
+
+# Expose a module-like alias so callers can use: uuid.uuid4()
+uuid = _uuid
+
+
+def uuid4():
+    """Return a UUID4 using the selected backend."""
+    return uuid.uuid4()

litellm/constants.py

@@ -812,6 +812,11 @@ BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[
 ]
 
 BEDROCK_CONVERSE_MODELS = [
+    "qwen.qwen3-coder-480b-a35b-v1:0",
+    "qwen.qwen3-235b-a22b-2507-v1:0",
+    "qwen.qwen3-coder-30b-a3b-v1:0",
+    "qwen.qwen3-32b-v1:0",
+    "deepseek.v3-v1:0",
     "openai.gpt-oss-20b-1:0",
     "openai.gpt-oss-120b-1:0",
     "anthropic.claude-opus-4-1-20250805-v1:0",

litellm/integrations/langfuse/langfuse.py

@@ -671,6 +671,7 @@ class LangFuseLogger:
 
             generation_id = None
             usage = None
+            usage_details = None
             if response_obj is not None:
                 if (
                     hasattr(response_obj, "id")
@@ -687,6 +688,11 @@ class LangFuseLogger:
                         "completion_tokens": _usage_obj.completion_tokens,
                         "total_cost": cost if self._supports_costs() else None,
                     }
+                    usage_details = LangfuseUsageDetails(input=_usage_obj.prompt_tokens,
+                                                        output=_usage_obj.completion_tokens,
+                                                        cache_creation_input_tokens=_usage_obj.get('cache_creation_input_tokens', 0),
+                                                        cache_read_input_tokens=_usage_obj.get('cache_read_input_tokens', 0))
+
             generation_name = clean_metadata.pop("generation_name", None)
             if generation_name is None:
                 # if `generation_name` is None, use sensible default values
@@ -719,6 +725,7 @@ class LangFuseLogger:
                 "input": input if not mask_input else "redacted-by-litellm",
                 "output": output if not mask_output else "redacted-by-litellm",
                 "usage": usage,
+                "usage_details": usage_details,
                 "metadata": log_requester_metadata(clean_metadata),
                 "level": level,
                 "version": clean_metadata.pop("version", None),

litellm/litellm_core_utils/litellm_logging.py

@@ -26,7 +26,6 @@ from typing import (
     cast,
 )
 
-import fastuuid as uuid
 from httpx import Response
 from pydantic import BaseModel
 
@@ -38,6 +37,7 @@ from litellm import (
     turn_off_message_logging,
 )
 from litellm._logging import _is_debugging_on, verbose_logger
+from litellm._uuid import uuid
 from litellm.batches.batch_utils import _handle_completed_batch
 from litellm.caching.caching import DualCache, InMemoryCache
 from litellm.caching.caching_handler import LLMCachingHandler

litellm/litellm_core_utils/streaming_handler.py

@@ -1619,11 +1619,12 @@ class CustomStreamWrapper:
                             completion_start_time=datetime.datetime.now()
                         )
                     ## LOGGING
-                    executor.submit(
-                        self.run_success_logging_and_cache_storage,
-                        response,
-                        cache_hit,
-                    )  # log response
+                    if not litellm.disable_streaming_logging:
+                        executor.submit(
+                            self.run_success_logging_and_cache_storage,
+                            response,
+                            cache_hit,
+                        )  # log response
                     choice = response.choices[0]
                     if isinstance(choice, StreamingChoices):
                         self.response_uptil_now += choice.delta.get("content", "") or ""

litellm/llms/anthropic/chat/transformation.py

@@ -804,7 +804,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             if content.get("citations") is not None:
                 if citations is None:
                     citations = []
-                citations.append(content["citations"])
+                citations.append(
+                    [
+                        {
+                            **citation,
+                            "supported_text": content.get("text", ""),
+                        }
+                        for citation in content["citations"]
+                    ]
+                )
         if thinking_blocks is not None:
             reasoning_content = ""
             for block in thinking_blocks:

litellm/llms/azure/chat/o_series_handler.py

@@ -4,7 +4,7 @@ Handler file for calls to Azure OpenAI's o1/o3 family of models
 Written separately to handle faking streaming for o1 and o3 models.
 """
 
-from typing import Any, Callable, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union
 
 import httpx
 
@@ -13,6 +13,9 @@ from litellm.types.utils import ModelResponse
 from ...openai.openai import OpenAIChatCompletion
 from ..common_utils import BaseAzureLLM
 
+if TYPE_CHECKING:
+    from aiohttp import ClientSession
+
 
 class AzureOpenAIO1ChatCompletion(BaseAzureLLM, OpenAIChatCompletion):
     def completion(
@@ -38,6 +41,7 @@ class AzureOpenAIO1ChatCompletion(BaseAzureLLM, OpenAIChatCompletion):
         organization: Optional[str] = None,
         custom_llm_provider: Optional[str] = None,
         drop_params: Optional[bool] = None,
+        shared_session: Optional["ClientSession"] = None,
     ):
         client = self.get_azure_openai_client(
             litellm_params=litellm_params,
@@ -69,4 +73,5 @@ class AzureOpenAIO1ChatCompletion(BaseAzureLLM, OpenAIChatCompletion):
             organization=organization,
             custom_llm_provider=custom_llm_provider,
             drop_params=drop_params,
+            shared_session=shared_session,
         )

litellm/llms/bedrock/common_utils.py

@@ -774,7 +774,7 @@ class CommonBatchFilesUtils:
         Returns:
             Unique job name (≤ 63 characters for Bedrock compatibility)
         """
-        import fastuuid as uuid
+        from litellm._uuid import uuid
         unique_id = str(uuid.uuid4())[:8]
         # Format: {prefix}-batch-{model}-{uuid}
         # Example: litellm-batch-claude-266c398e

litellm/llms/custom_httpx/http_handler.py

@@ -167,6 +167,7 @@ class AsyncHTTPHandler:
         concurrent_limit=1000,
         client_alias: Optional[str] = None,  # name for client in logs
         ssl_verify: Optional[VerifyTypes] = None,
+        shared_session: Optional["ClientSession"] = None,
     ):
         self.timeout = timeout
         self.event_hooks = event_hooks
@@ -175,6 +176,7 @@ class AsyncHTTPHandler:
             concurrent_limit=concurrent_limit,
             event_hooks=event_hooks,
             ssl_verify=ssl_verify,
+            shared_session=shared_session,
         )
         self.client_alias = client_alias
 
@@ -184,6 +186,7 @@ class AsyncHTTPHandler:
         concurrent_limit: int,
         event_hooks: Optional[Mapping[str, List[Callable[..., Any]]]],
         ssl_verify: Optional[VerifyTypes] = None,
+        shared_session: Optional["ClientSession"] = None,
     ) -> httpx.AsyncClient:
         # Get unified SSL configuration
         ssl_config = get_ssl_configuration(ssl_verify)
@@ -199,6 +202,7 @@ class AsyncHTTPHandler:
         transport = AsyncHTTPHandler._create_async_transport(
             ssl_context=ssl_config if isinstance(ssl_config, ssl.SSLContext) else None,
             ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
+            shared_session=shared_session,
         )
 
         return httpx.AsyncClient(
@@ -260,7 +264,6 @@ class AsyncHTTPHandler:
         files: Optional[RequestFiles] = None,
         content: Any = None,
     ):
-
         start_time = time.time()
         try:
             if timeout is None:
@@ -523,7 +526,9 @@ class AsyncHTTPHandler:
 
     @staticmethod
     def _create_async_transport(
-        ssl_context: Optional[ssl.SSLContext] = None, ssl_verify: Optional[bool] = None
+        ssl_context: Optional[ssl.SSLContext] = None,
+        ssl_verify: Optional[bool] = None,
+        shared_session: Optional["ClientSession"] = None,
     ) -> Optional[Union[LiteLLMAiohttpTransport, AsyncHTTPTransport]]:
         """
         - Creates a transport for httpx.AsyncClient
@@ -544,7 +549,9 @@ class AsyncHTTPHandler:
         #########################################################
         if AsyncHTTPHandler._should_use_aiohttp_transport():
             return AsyncHTTPHandler._create_aiohttp_transport(
-                ssl_context=ssl_context, ssl_verify=ssl_verify
+                ssl_context=ssl_context,
+                ssl_verify=ssl_verify,
+                shared_session=shared_session,
             )
 
         #########################################################
@@ -612,6 +619,7 @@ class AsyncHTTPHandler:
     def _create_aiohttp_transport(
         ssl_verify: Optional[bool] = None,
         ssl_context: Optional[ssl.SSLContext] = None,
+        shared_session: Optional["ClientSession"] = None,
     ) -> LiteLLMAiohttpTransport:
         """
         Creates an AiohttpTransport with RequestNotRead error handling
@@ -635,6 +643,18 @@ class AsyncHTTPHandler:
             trust_env = True
 
         verbose_logger.debug("Creating AiohttpTransport...")
+
+        # Use shared session if provided and valid
+        if shared_session is not None and not shared_session.closed:
+            verbose_logger.debug(
+                f"SHARED SESSION: Reusing existing ClientSession (ID: {id(shared_session)})"
+            )
+            return LiteLLMAiohttpTransport(client=shared_session)
+
+        # Create new session only if none provided or existing one is invalid
+        verbose_logger.debug(
+            "NEW SESSION: Creating new ClientSession (no shared session provided)"
+        )
         return LiteLLMAiohttpTransport(
             client=lambda: ClientSession(
                 connector=TCPConnector(**connector_kwargs),
@@ -915,12 +935,13 @@ class HTTPHandler:
         if litellm.force_ipv4:
             return HTTPTransport(local_address="0.0.0.0")
         else:
-            return None
+            return getattr(litellm, 'sync_transport', None)
 
 
 def get_async_httpx_client(
     llm_provider: Union[LlmProviders, httpxSpecialProvider],
     params: Optional[dict] = None,
+    shared_session: Optional["ClientSession"] = None,
 ) -> AsyncHTTPHandler:
     """
     Retrieves the async HTTP client from the cache
@@ -942,10 +963,12 @@ def get_async_httpx_client(
         return _cached_client
 
     if params is not None:
+        params["shared_session"] = shared_session
         _new_client = AsyncHTTPHandler(**params)
     else:
         _new_client = AsyncHTTPHandler(
-            timeout=httpx.Timeout(timeout=600.0, connect=5.0)
+            timeout=httpx.Timeout(timeout=600.0, connect=5.0),
+            shared_session=shared_session,
         )
 
     litellm.in_memory_llm_clients_cache.set_cache(

litellm/llms/custom_httpx/llm_http_handler.py

@@ -88,6 +88,7 @@ from litellm.utils import (
 )
 
 if TYPE_CHECKING:
+    from aiohttp import ClientSession
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
     from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
 
@@ -236,11 +237,16 @@ class BaseLLMHTTPHandler:
         client: Optional[AsyncHTTPHandler] = None,
         json_mode: bool = False,
         signed_json_body: Optional[bytes] = None,
+        shared_session: Optional["ClientSession"] = None,
     ):
         if client is None:
+            verbose_logger.debug(
+                f"Creating HTTP client with shared_session: {id(shared_session) if shared_session else None}"
+            )
             async_httpx_client = get_async_httpx_client(
                 llm_provider=litellm.LlmProviders(custom_llm_provider),
                 params={"ssl_verify": litellm_params.get("ssl_verify", None)},
+                shared_session=shared_session,
             )
         else:
             async_httpx_client = client
@@ -290,6 +296,7 @@ class BaseLLMHTTPHandler:
         headers: Optional[Dict[str, Any]] = None,
         client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
         provider_config: Optional[BaseConfig] = None,
+        shared_session: Optional["ClientSession"] = None,
     ):
         json_mode: bool = optional_params.pop("json_mode", False)
         extra_body: Optional[dict] = optional_params.pop("extra_body", None)
@@ -469,7 +476,7 @@ class BaseLLMHTTPHandler:
 
         if client is None or not isinstance(client, HTTPHandler):
             sync_httpx_client = _get_httpx_client(
-                params={"ssl_verify": litellm_params.get("ssl_verify", None)}
+                params={"ssl_verify": litellm_params.get("ssl_verify", None)},
             )
         else:
             sync_httpx_client = client
@@ -2283,7 +2290,7 @@ class BaseLLMHTTPHandler:
                     e=e,
                     provider_config=provider_config,
                 )
-        
+
         # Store the upload URL in litellm_params for the transformation method
         litellm_params_with_url = dict(litellm_params)
         litellm_params_with_url["upload_url"] = api_base
@@ -2574,11 +2581,11 @@ class BaseLLMHTTPHandler:
                     "url": transformed_request["url"],
                     "headers": transformed_request["headers"],
                 }
-                
+
                 # Only add data for non-GET requests
                 if method != "get" and transformed_request.get("data") is not None:
                     request_kwargs["data"] = transformed_request["data"]
-                
+
                 batch_response = getattr(sync_httpx_client, method)(**request_kwargs)
             elif isinstance(transformed_request, dict) and api_base:
                 # For other providers that use JSON requests
@@ -2743,12 +2750,14 @@ class BaseLLMHTTPHandler:
                     "url": transformed_request["url"],
                     "headers": transformed_request["headers"],
                 }
-                
+
                 # Only add data for non-GET requests
                 if method != "get" and transformed_request.get("data") is not None:
                     request_kwargs["data"] = transformed_request["data"]
-                
-                batch_response = await getattr(async_httpx_client, method)(**request_kwargs)
+
+                batch_response = await getattr(async_httpx_client, method)(
+                    **request_kwargs
+                )
             elif isinstance(transformed_request, dict) and api_base:
                 # For other providers that use JSON requests
                 batch_response = await async_httpx_client.get(

litellm/llms/openai/common_utils.py

@@ -5,12 +5,15 @@ Common helpers / utils across al OpenAI endpoints
 import hashlib
 import json
 import ssl
-from typing import Any, Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, TYPE_CHECKING, Union
 
 import httpx
 import openai
 from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
 
+if TYPE_CHECKING:
+    from aiohttp import ClientSession
+
 import litellm
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
@@ -194,7 +197,9 @@ class BaseOpenAILLM:
         return param_names
 
     @staticmethod
-    def _get_async_http_client() -> Optional[httpx.AsyncClient]:
+    def _get_async_http_client(
+        shared_session: Optional["ClientSession"] = None,
+    ) -> Optional[httpx.AsyncClient]:
         if litellm.aclient_session is not None:
             return litellm.aclient_session
 
@@ -205,8 +210,11 @@ class BaseOpenAILLM:
             limits=httpx.Limits(max_connections=1000, max_keepalive_connections=100),
             verify=ssl_config,
             transport=AsyncHTTPHandler._create_async_transport(
-                ssl_context=ssl_config if isinstance(ssl_config, ssl.SSLContext) else None,
+                ssl_context=ssl_config
+                if isinstance(ssl_config, ssl.SSLContext)
+                else None,
                 ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
+                shared_session=shared_session,
             ),
             follow_redirects=True,
         )
@@ -215,10 +223,10 @@ class BaseOpenAILLM:
     def _get_sync_http_client() -> Optional[httpx.Client]:
         if litellm.client_session is not None:
             return litellm.client_session
-        
+
         # Get unified SSL configuration
         ssl_config = get_ssl_configuration()
-        
+
         return httpx.Client(
             limits=httpx.Limits(max_connections=1000, max_keepalive_connections=100),
             verify=ssl_config,

litellm/llms/openai/openai.py

@@ -10,12 +10,16 @@ from typing import (
     List,
     Literal,
     Optional,
+    TYPE_CHECKING,
     Union,
     cast,
 )
 from urllib.parse import urlparse
 
 import httpx
+
+if TYPE_CHECKING:
+    from aiohttp import ClientSession
 import openai
 from openai import AsyncOpenAI, OpenAI
 from openai.types.beta.assistant_deleted import AssistantDeleted
@@ -355,6 +359,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         max_retries: Optional[int] = DEFAULT_MAX_RETRIES,
         organization: Optional[str] = None,
         client: Optional[Union[OpenAI, AsyncOpenAI]] = None,
+        shared_session: Optional["ClientSession"] = None,
     ) -> Optional[Union[OpenAI, AsyncOpenAI]]:
         client_initialization_params: Dict = locals()
         if client is None:
@@ -379,7 +384,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                 _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI(
                     api_key=api_key,
                     base_url=api_base,
-                    http_client=OpenAIChatCompletion._get_async_http_client(),
+                    http_client=OpenAIChatCompletion._get_async_http_client(
+                        shared_session=shared_session
+                    ),
                     timeout=timeout,
                     max_retries=max_retries,
                     organization=organization,
@@ -522,8 +529,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         organization: Optional[str] = None,
         custom_llm_provider: Optional[str] = None,
         drop_params: Optional[bool] = None,
+        shared_session: Optional["ClientSession"] = None,
     ):
-        super().completion()
+        super().completion(shared_session=shared_session)
         try:
             fake_stream: bool = False
             inference_params = optional_params.copy()
@@ -606,6 +614,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                                 organization=organization,
                                 drop_params=drop_params,
                                 fake_stream=fake_stream,
+                                shared_session=shared_session,
                             )
 
                     data = provider_config.transform_request(
@@ -771,6 +780,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         drop_params: Optional[bool] = None,
         stream_options: Optional[dict] = None,
         fake_stream: bool = False,
+        shared_session: Optional["ClientSession"] = None,
     ):
         response = None
         data = await provider_config.async_transform_request(
@@ -793,6 +803,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                     max_retries=max_retries,
                     organization=organization,
                     client=client,
+                    shared_session=shared_session,
                 )
 
                 ## LOGGING

litellm/llms/vertex_ai/gemini/transformation.py

@@ -537,7 +537,11 @@ def sync_transform_request_body(
             logging_obj=logging_obj,
         )
     else:  # [TODO] implement context caching for gemini as well
-        cached_content = optional_params.pop("cached_content", None)
+        cached_content = None
+        if "cached_content" in optional_params:
+            cached_content = optional_params.pop("cached_content")
+        elif "cachedContent" in optional_params:
+            cached_content = optional_params.pop("cachedContent")
 
     return _transform_request_body(
         messages=messages,
@@ -584,7 +588,11 @@ async def async_transform_request_body(
             logging_obj=logging_obj,
         )
     else:  # [TODO] implement context caching for gemini as well
-        cached_content = optional_params.pop("cached_content", None)
+        cached_content = None
+        if "cached_content" in optional_params:
+            cached_content = optional_params.pop("cached_content")
+        elif "cachedContent" in optional_params:
+            cached_content = optional_params.pop("cachedContent")
 
     return _transform_request_body(
         messages=messages,
@@ -649,5 +657,3 @@ def _transform_system_message(
         return SystemInstructions(parts=system_content_blocks), messages
 
     return None, messages
-
-

litellm/llms/vertex_ai/vertex_llm_base.py

@@ -271,17 +271,11 @@ class VertexBase:
 
     def is_using_v1beta1_features(self, optional_params: dict) -> bool:
         """
-        VertexAI only supports ContextCaching on v1beta1
-
         use this helper to decide if request should be sent to v1 or v1beta1
 
-        Returns v1beta1 if context caching is enabled
-        Returns v1 in all other cases
+        Returns true if any beta feature is enabled
+        Returns false in all other cases
         """
-        if "cached_content" in optional_params:
-            return True
-        if "CachedContent" in optional_params:
-            return True
         return False
 
     def _check_custom_proxy(

litellm/main.py

@@ -36,8 +36,12 @@ from typing import (
     Union,
     cast,
     get_args,
+    TYPE_CHECKING,
 )
 
+if TYPE_CHECKING:
+    from aiohttp import ClientSession
+
 import dotenv
 import httpx
 import openai
@@ -374,6 +378,8 @@ async def acompletion(
     # Optional liteLLM function params
     thinking: Optional[AnthropicThinkingParam] = None,
     web_search_options: Optional[OpenAIWebSearchOptions] = None,
+    # Session management
+    shared_session: Optional["ClientSession"] = None,
     **kwargs,
 ) -> Union[ModelResponse, CustomStreamWrapper]:
     """
@@ -466,6 +472,16 @@ async def acompletion(
     #########################################################
     #########################################################
 
+    # Log shared session usage
+    if shared_session is not None:
+        verbose_logger.debug(
+            f"🔄 SHARED SESSION: acompletion called with shared_session (ID: {id(shared_session)})"
+        )
+    else:
+        verbose_logger.debug(
+            "🔄 NO SHARED SESSION: acompletion called without shared_session"
+        )
+
     # Adjusted to use explicit arguments instead of *args and **kwargs
     completion_kwargs = {
         "model": model,
@@ -506,6 +522,7 @@ async def acompletion(
         "acompletion": True,  # assuming this is a required parameter
         "thinking": thinking,
         "web_search_options": web_search_options,
+        "shared_session": shared_session,
     }
     if custom_llm_provider is None:
         _, custom_llm_provider, _, _ = get_llm_provider(
@@ -930,6 +947,8 @@ def completion(  # type: ignore # noqa: PLR0915
     model_list: Optional[list] = None,  # pass in a list of api_base,keys, etc.
     # Optional liteLLM function params
     thinking: Optional[AnthropicThinkingParam] = None,
+    # Session management
+    shared_session: Optional["ClientSession"] = None,
     **kwargs,
 ) -> Union[ModelResponse, CustomStreamWrapper]:
     """
@@ -1004,15 +1023,7 @@ def completion(  # type: ignore # noqa: PLR0915
     provider_specific_header = cast(
         Optional[ProviderSpecificHeader], kwargs.get("provider_specific_header", None)
     )
-    # Properly merge headers with priority: request headers > extra_headers > global litellm.headers
-    headers = {}
-    if litellm.headers is not None and isinstance(litellm.headers, dict):
-        headers.update(litellm.headers)
-    if extra_headers is not None and isinstance(extra_headers, dict):
-        headers.update(extra_headers)
-    request_headers = kwargs.get("headers", None)
-    if request_headers is not None and isinstance(request_headers, dict):
-        headers.update(request_headers)
+    headers = kwargs.get("headers", None) or extra_headers
 
     ensure_alternating_roles: Optional[bool] = kwargs.get(
         "ensure_alternating_roles", None
@@ -1023,6 +1034,10 @@ def completion(  # type: ignore # noqa: PLR0915
     assistant_continue_message: Optional[ChatCompletionAssistantMessage] = kwargs.get(
         "assistant_continue_message", None
     )
+    if headers is None:
+        headers = {}
+    if extra_headers is not None:
+        headers.update(extra_headers)
     num_retries = kwargs.get(
         "num_retries", None
     )  ## alt. param for 'max_retries'. Use this to pass retries w/ instructor.
@@ -1079,6 +1094,7 @@ def completion(  # type: ignore # noqa: PLR0915
             prompt_id=prompt_id, non_default_params=non_default_params
         )
     ):
+
         (
             model,
             messages,
@@ -1431,7 +1447,8 @@ def completion(  # type: ignore # noqa: PLR0915
                 "azure_ad_token_provider", None
             )
 
-            # Use the consolidated headers that were already merged at the top of the function
+            headers = headers or litellm.headers
+
             if extra_headers is not None:
                 optional_params["extra_headers"] = extra_headers
             if max_retries is not None:
@@ -1598,6 +1615,7 @@ def completion(  # type: ignore # noqa: PLR0915
                     logging_obj=logging,
                     optional_params=optional_params,
                     litellm_params=litellm_params,
+                    shared_session=shared_session,
                     timeout=timeout,  # type: ignore
                     client=client,
                     custom_llm_provider=custom_llm_provider,
@@ -1644,6 +1662,7 @@ def completion(  # type: ignore # noqa: PLR0915
                     logging_obj=logging,
                     optional_params=optional_params,
                     litellm_params=litellm_params,
+                    shared_session=shared_session,
                     timeout=timeout,  # type: ignore
                     client=client,  # pass AsyncOpenAI, OpenAI client
                     custom_llm_provider=custom_llm_provider,
@@ -1696,7 +1715,8 @@ def completion(  # type: ignore # noqa: PLR0915
                 or get_secret("OPENAI_API_KEY")
             )
 
-            # Use the consolidated headers that were already merged at the top of the function
+            headers = headers or litellm.headers
+
             if extra_headers is not None:
                 optional_params["extra_headers"] = extra_headers
 
@@ -1772,6 +1792,7 @@ def completion(  # type: ignore # noqa: PLR0915
                     logging_obj=logging,
                     optional_params=optional_params,
                     litellm_params=litellm_params,
+                    shared_session=shared_session,
                     timeout=timeout,  # type: ignore
                     client=client,
                     custom_llm_provider=custom_llm_provider,
@@ -1801,6 +1822,7 @@ def completion(  # type: ignore # noqa: PLR0915
                     logging_obj=logging,
                     optional_params=optional_params,
                     litellm_params=litellm_params,
+                    shared_session=shared_session,
                     timeout=timeout,
                     client=client,
                     custom_llm_provider=custom_llm_provider,
@@ -1831,6 +1853,7 @@ def completion(  # type: ignore # noqa: PLR0915
                     logging_obj=logging,
                     optional_params=optional_params,
                     litellm_params=litellm_params,
+                    shared_session=shared_session,
                     timeout=timeout,  # type: ignore
                     client=client,
                     custom_llm_provider=custom_llm_provider,
@@ -1882,6 +1905,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 model_response=model_response,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 custom_llm_provider=custom_llm_provider,
                 timeout=timeout,
                 headers=headers,
@@ -1955,6 +1979,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 logging_obj=logging,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 timeout=timeout,
                 client=client,
                 custom_llm_provider=custom_llm_provider,
@@ -2034,6 +2059,7 @@ def completion(  # type: ignore # noqa: PLR0915
 
             try:
                 if use_base_llm_http_handler:
+
                     response = base_llm_http_handler.completion(
                         model=model,
                         messages=messages,
@@ -2045,6 +2071,7 @@ def completion(  # type: ignore # noqa: PLR0915
                         optional_params=optional_params,
                         timeout=timeout,
                         litellm_params=litellm_params,
+                        shared_session=shared_session,
                         acompletion=acompletion,
                         stream=stream,
                         api_key=api_key,
@@ -2071,6 +2098,7 @@ def completion(  # type: ignore # noqa: PLR0915
                         client=client,  # pass AsyncOpenAI, OpenAI client
                         organization=organization,
                         custom_llm_provider=custom_llm_provider,
+                        shared_session=shared_session,
                     )
             except Exception as e:
                 ## LOGGING - log the original exception returned
@@ -2111,6 +2139,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 optional_params=optional_params,
                 timeout=timeout,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 acompletion=acompletion,
                 stream=stream,
                 api_key=api_key,
@@ -2198,6 +2227,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 model_response=model_response,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 custom_llm_provider="clarifai",
                 timeout=timeout,
                 headers=headers,
@@ -2243,6 +2273,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 model_response=model_response,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 custom_llm_provider="anthropic_text",
                 timeout=timeout,
                 headers=headers,
@@ -2412,8 +2443,12 @@ def completion(  # type: ignore # noqa: PLR0915
                 or "https://api.cohere.ai/v1/chat"
             )
 
-            # Use the consolidated headers that were already merged at the top of the function
-            # No need for additional merging here as it's already done
+            headers = headers or litellm.headers or {}
+            if headers is None:
+                headers = {}
+
+            if extra_headers is not None:
+                headers.update(extra_headers)
 
             response = base_llm_http_handler.completion(
                 model=model,
@@ -2424,6 +2459,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 model_response=model_response,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 custom_llm_provider="cohere_chat",
                 timeout=timeout,
                 headers=headers,
@@ -2509,10 +2545,15 @@ def completion(  # type: ignore # noqa: PLR0915
             )
         elif custom_llm_provider == "compactifai":
             api_key = (
-                api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key
+                api_key
+                or get_secret_str("COMPACTIFAI_API_KEY")
+                or litellm.api_key
             )
 
-            api_base = api_base or "https://api.compactif.ai/v1"
+            api_base = (
+                api_base
+                or "https://api.compactif.ai/v1"
+            )
 
             ## COMPLETION CALL
             response = base_llm_http_handler.completion(
@@ -2686,6 +2727,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 model_response=model_response,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 custom_llm_provider="openrouter",
                 timeout=timeout,
                 headers=headers,
@@ -2748,6 +2790,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 model_response=model_response,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 custom_llm_provider="vercel_ai_gateway",
                 timeout=timeout,
                 headers=headers,
@@ -3098,9 +3141,9 @@ def completion(  # type: ignore # noqa: PLR0915
                     "aws_region_name" not in optional_params
                     or optional_params["aws_region_name"] is None
                 ):
-                    optional_params[
-                        "aws_region_name"
-                    ] = aws_bedrock_client.meta.region_name
+                    optional_params["aws_region_name"] = (
+                        aws_bedrock_client.meta.region_name
+                    )
 
             bedrock_route = BedrockModelInfo.get_bedrock_route(model)
             if bedrock_route == "converse":
@@ -3234,6 +3277,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 model_response=model_response,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 custom_llm_provider="watsonx_text",
                 timeout=timeout,
                 headers=headers,
@@ -3287,6 +3331,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 model_response=model_response,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 custom_llm_provider="ollama",
                 timeout=timeout,
                 headers=headers,
@@ -3320,6 +3365,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 model_response=model_response,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 custom_llm_provider="ollama_chat",
                 timeout=timeout,
                 headers=headers,
@@ -3340,6 +3386,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 model_response=model_response,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 custom_llm_provider=custom_llm_provider,
                 timeout=timeout,
                 headers=headers,
@@ -3372,6 +3419,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 model_response=model_response,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 custom_llm_provider="cloudflare",
                 timeout=timeout,
                 headers=headers,
@@ -3425,6 +3473,7 @@ def completion(  # type: ignore # noqa: PLR0915
                     logging_obj=logging,
                     optional_params=optional_params,
                     litellm_params=litellm_params,
+                    shared_session=shared_session,
                     timeout=timeout,  # type: ignore
                     client=client,
                     custom_llm_provider=custom_llm_provider,
@@ -3442,6 +3491,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 )
                 raise e
         elif custom_llm_provider == "gradient_ai":
+
             api_base = litellm.api_base or api_base
             response = base_llm_http_handler.completion(
                 model=model,
@@ -3452,6 +3502,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 model_response=model_response,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
+                shared_session=shared_session,
                 custom_llm_provider="gradient_ai",
                 timeout=timeout,
                 headers=headers,
@@ -3605,7 +3656,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 async_fn=acompletion, stream=stream, custom_llm=custom_handler
             )
 
-            headers = headers or litellm.headers
+            headers = headers or litellm.headers or {}
 
             ## CALL FUNCTION
             response = handler_fn(
@@ -3801,7 +3852,7 @@ def embedding(
     *,
     aembedding: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, EmbeddingResponse]:
+) -> Coroutine[Any, Any, EmbeddingResponse]: 
     ...
 
 
@@ -3827,7 +3878,7 @@ def embedding(
     *,
     aembedding: Literal[False] = False,
     **kwargs,
-) -> EmbeddingResponse:
+) -> EmbeddingResponse: 
     ...
 
 # fmt: on
@@ -4138,8 +4189,10 @@ def embedding(  # noqa: PLR0915
                 or litellm.api_key
             )
 
-            # Use the consolidated headers that were already merged at the top of the function
-            # No need for additional merging here as it's already done
+            if extra_headers is not None and isinstance(extra_headers, dict):
+                headers = extra_headers
+            else:
+                headers = {}
 
             response = base_llm_http_handler.embedding(
                 model=model,
@@ -5099,9 +5152,9 @@ def adapter_completion(
     new_kwargs = translation_obj.translate_completion_input_params(kwargs=kwargs)
 
     response: Union[ModelResponse, CustomStreamWrapper] = completion(**new_kwargs)  # type: ignore
-    translated_response: Optional[
-        Union[BaseModel, AdapterCompletionStreamWrapper]
-    ] = None
+    translated_response: Optional[Union[BaseModel, AdapterCompletionStreamWrapper]] = (
+        None
+    )
     if isinstance(response, ModelResponse):
         translated_response = translation_obj.translate_completion_output_params(
             response=response
@@ -6089,9 +6142,9 @@ def stream_chunk_builder(  # noqa: PLR0915
         ]
 
         if len(content_chunks) > 0:
-            response["choices"][0]["message"][
-                "content"
-            ] = processor.get_combined_content(content_chunks)
+            response["choices"][0]["message"]["content"] = (
+                processor.get_combined_content(content_chunks)
+            )
 
         thinking_blocks = [
             chunk
@@ -6102,9 +6155,9 @@ def stream_chunk_builder(  # noqa: PLR0915
         ]
 
         if len(thinking_blocks) > 0:
-            response["choices"][0]["message"][
-                "thinking_blocks"
-            ] = processor.get_combined_thinking_content(thinking_blocks)
+            response["choices"][0]["message"]["thinking_blocks"] = (
+                processor.get_combined_thinking_content(thinking_blocks)
+            )
 
         reasoning_chunks = [
             chunk
@@ -6115,9 +6168,9 @@ def stream_chunk_builder(  # noqa: PLR0915
         ]
 
         if len(reasoning_chunks) > 0:
-            response["choices"][0]["message"][
-                "reasoning_content"
-            ] = processor.get_combined_reasoning_content(reasoning_chunks)
+            response["choices"][0]["message"]["reasoning_content"] = (
+                processor.get_combined_reasoning_content(reasoning_chunks)
+            )
 
         audio_chunks = [
             chunk

litellm/model_prices_and_context_window_backup.json

@@ -560,10 +560,14 @@
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -824,10 +828,14 @@
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -4787,10 +4795,14 @@
         "cache_creation_input_token_cost_above_1hr": 6e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "anthropic",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -7191,6 +7203,18 @@
         "supports_prompt_caching": true,
         "supports_tool_choice": true
     },
+    "deepseek.v3-v1:0": {
+        "input_cost_per_token": 5.8e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 163840,
+        "max_output_tokens": 81920,
+        "max_tokens": 163840,
+        "mode": "chat",
+        "output_cost_per_token": 1.68e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },     
     "dolphin": {
         "input_cost_per_token": 5e-07,
         "litellm_provider": "nlp_cloud",
@@ -7547,10 +7571,14 @@
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -12332,6 +12360,7 @@
         "supports_response_schema": true,
         "supports_system_messages": true,
         "supports_tool_choice": true,
+        "supports_service_tier": true,
         "supports_vision": true
     },
     "gpt-5-chat": {
@@ -12479,6 +12508,7 @@
         "cache_read_input_token_cost_flex": 2.5e-09,
         "input_cost_per_token": 5e-08,
         "input_cost_per_token_flex": 2.5e-08,
+        "input_cost_per_token_priority": 2.5e-06,
         "litellm_provider": "openai",
         "max_input_tokens": 400000,
         "max_output_tokens": 128000,
@@ -15452,7 +15482,7 @@
     },
     "o4-mini": {
         "cache_read_input_token_cost": 2.75e-07,
-        "cache_read_input_token_cost_flex": 1.38e-07,
+        "cache_read_input_token_cost_flex": 1.375e-07,
         "cache_read_input_token_cost_priority": 5e-07,
         "input_cost_per_token": 1.1e-06,
         "input_cost_per_token_flex": 5.5e-07,
@@ -16155,10 +16185,12 @@
     "openrouter/anthropic/claude-sonnet-4": {
         "input_cost_per_image": 0.0048,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
         "litellm_provider": "openrouter",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "supports_assistant_prefill": true,
@@ -17500,6 +17532,54 @@
         "mode": "chat",
         "output_cost_per_token": 2.8e-07
     },
+    "qwen.qwen3-coder-480b-a35b-v1:0": {
+        "input_cost_per_token": 2.2e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262000,
+        "max_output_tokens": 65536,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 1.8e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "qwen.qwen3-235b-a22b-2507-v1:0": {
+        "input_cost_per_token": 2.2e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 131072,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 8.8e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "qwen.qwen3-coder-30b-a3b-v1:0": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 131072,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 6.0e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    }, 
+    "qwen.qwen3-32b-v1:0": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 16384,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 6.0e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },      
     "recraft/recraftv2": {
         "litellm_provider": "recraft",
         "mode": "image_generation",
@@ -19018,10 +19098,14 @@
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -20354,10 +20438,14 @@
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "vertex_ai-anthropic_models",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -20380,10 +20468,14 @@
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "vertex_ai-anthropic_models",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -21187,6 +21279,30 @@
             "/v1/audio/transcriptions"
         ]
     },
+    "vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "vertex_ai-qwen_models",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "vertex_ai-qwen_models",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
     "xai/grok-2": {
         "input_cost_per_token": 2e-06,
         "litellm_provider": "xai",
@@ -21439,6 +21555,35 @@
         "supports_tool_choice": true,
         "supports_web_search": true
     },
+    "xai/grok-4-fast-reasoning": {
+        "litellm_provider": "xai",
+        "max_input_tokens": 2e6,
+        "max_output_tokens": 2e6,
+        "max_tokens": 2e6,
+        "mode": "chat",
+        "input_cost_per_token": 0.2e-06,
+        "output_cost_per_token": 0.5e-06,
+        "cache_read_input_token_cost": 0.05e-06,
+        "source": "https://docs.x.ai/docs/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_web_search": true
+    },
+    "xai/grok-4-fast-non-reasoning": {
+        "litellm_provider": "xai",
+        "max_input_tokens": 2e6,
+        "max_output_tokens": 2e6,
+        "cache_read_input_token_cost": 0.05e-06,
+        "max_tokens": 2e6,
+        "mode": "chat",
+        "input_cost_per_token": 0.2e-06,
+        "output_cost_per_token": 0.5e-06,
+        "source": "https://docs.x.ai/docs/models",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_web_search": true
+    },
     "xai/grok-4-0709": {
         "input_cost_per_token": 3e-06,
         "litellm_provider": "xai",

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -421,7 +421,6 @@ class MCPServerManager:
         self,
         server: MCPServer,
         mcp_auth_header: Optional[str] = None,
-        add_prefix: bool = True,
     ) -> List[MCPTool]:
         """
         Helper method to get tools from a single MCP server with prefixed names.
@@ -446,11 +445,9 @@ class MCPServerManager:
 
             tools = await self._fetch_tools_with_timeout(client, server.name)
 
-            prefixed_or_original_tools = self._create_prefixed_tools(
-                tools, server, add_prefix=add_prefix
-            )
+            prefixed_tools = self._create_prefixed_tools(tools, server)
 
-            return prefixed_or_original_tools
+            return prefixed_tools
 
         except Exception as e:
             verbose_logger.warning(
@@ -519,7 +516,7 @@ class MCPServerManager:
             return []
 
     def _create_prefixed_tools(
-        self, tools: List[MCPTool], server: MCPServer, add_prefix: bool = True
+        self, tools: List[MCPTool], server: MCPServer
     ) -> List[MCPTool]:
         """
         Create prefixed tools and update tool mapping.
@@ -537,16 +534,14 @@ class MCPServerManager:
         for tool in tools:
             prefixed_name = add_server_prefix_to_tool_name(tool.name, prefix)
 
-            name_to_use = prefixed_name if add_prefix else tool.name
-
-            tool_obj = MCPTool(
-                name=name_to_use,
+            prefixed_tool = MCPTool(
+                name=prefixed_name,
                 description=tool.description,
                 inputSchema=tool.inputSchema,
             )
-            prefixed_tools.append(tool_obj)
+            prefixed_tools.append(prefixed_tool)
 
-            # Update tool to server mapping for resolution (support both forms)
+            # Update tool to server mapping with both original and prefixed names
             self.tool_name_to_mcp_server_name_mapping[tool.name] = prefix
             self.tool_name_to_mcp_server_name_mapping[prefixed_name] = prefix
 

litellm/proxy/_experimental/mcp_server/rest_endpoints.py

@@ -73,7 +73,6 @@ if MCP_AVAILABLE:
         tools = await global_mcp_server_manager._get_tools_from_server(
             server=server,
             mcp_auth_header=server_auth_header,
-            add_prefix=False,
         )
         return _create_tool_response_objects(tools, server.mcp_info)
 

litellm/proxy/_experimental/mcp_server/server.py

@@ -384,9 +384,6 @@ if MCP_AVAILABLE:
                 allowed_mcp_servers=allowed_mcp_servers,
             )
 
-        # Decide whether to add prefix based on number of allowed servers
-        add_prefix = not (len(allowed_mcp_servers) == 1)
-
         # Get tools from each allowed server
         all_tools = []
         for server_id in allowed_mcp_servers:
@@ -409,7 +406,6 @@ if MCP_AVAILABLE:
                 tools = await global_mcp_server_manager._get_tools_from_server(
                     server=server,
                     mcp_auth_header=server_auth_header,
-                    add_prefix=add_prefix,
                 )
                 all_tools.extend(tools)
                 verbose_logger.debug(
@@ -641,35 +637,27 @@ if MCP_AVAILABLE:
         # Server names can contain slashes (e.g., "custom_solutions/user_123")
         mcp_path_match = re.match(r"^/mcp/([^?#]+)(?:\?.*)?(?:#.*)?$", path)
         if mcp_path_match:
-            mcp_servers_str = mcp_path_match.group(1)
-            optional_path = mcp_path_match.group(2)
-
-            if mcp_servers_str:
-                # First, try to split by comma for comma-separated lists
-                if "," in mcp_servers_str:
-                    # For comma-separated lists, we need to handle the case where the last item
-                    # might include the path (e.g., "zapier,group1/tools" -> ["zapier", "group1/tools"])
-                    parts = [s.strip() for s in mcp_servers_str.split(",") if s.strip()]
-
-                    # If there's an optional path AND the last part contains a slash that matches the optional path,
-                    # remove the path portion from the last server name
-                    if optional_path and len(parts) > 0 and "/" in parts[-1]:
-                        last_part = parts[-1]
-                        # Check if the last part ends with the optional path
-                        if optional_path and last_part.endswith(
-                            optional_path.lstrip("/")
-                        ):
-                            # Remove the path portion from the last server name
-                            parts[-1] = last_part[: -len(optional_path.lstrip("/"))]
-
-                    mcp_servers_from_path = parts
+            servers_and_path = mcp_path_match.group(1)
+            
+            if servers_and_path:
+                # Check if it contains commas (comma-separated servers)
+                if ',' in servers_and_path:
+                    # For comma-separated, look for a path at the end
+                    # Common patterns: /tools, /chat/completions, etc.
+                    path_match = re.search(r'/([^/,]+(?:/[^/,]+)*)$', servers_and_path)
+                    if path_match:
+                        # Path found at the end, remove it from servers
+                        path_part = '/' + path_match.group(1)
+                        servers_part = servers_and_path[:-len(path_part)]
+                        mcp_servers_from_path = [s.strip() for s in servers_part.split(',') if s.strip()]
+                    else:
+                        # No path, just comma-separated servers
+                        mcp_servers_from_path = [s.strip() for s in servers_and_path.split(',') if s.strip()]
                 else:
-                    # For single server, it might be just a name or contain slashes
-                    # We need to determine where the server name ends and the path begins
-                    # This is tricky - let's use the original logic but handle comma cases differently
-                    single_server_match = re.match(
-                        r"^([^/]+(?:/[^/]+)?)(?:/.*)?$", mcp_servers_str
-                    )
+                    # Single server case - use regex approach for server/path separation
+                    # This handles cases like "custom_solutions/user_123/chat/completions"
+                    # where we want to extract "custom_solutions/user_123" as the server name
+                    single_server_match = re.match(r"^([^/]+(?:/[^/]+)?)(?:/.*)?$", servers_and_path)
                     if single_server_match:
                         server_name = single_server_match.group(1)
                         mcp_servers_from_path = [server_name]

litellm/proxy/auth/auth_checks.py

@@ -469,7 +469,6 @@ async def get_end_user_object(
     # check if in cache
     cached_user_obj = await user_api_key_cache.async_get_cache(key=_key)
     if cached_user_obj is not None:
-        # Convert cached dict to LiteLLM_EndUserTable instance
         return_obj = LiteLLM_EndUserTable(**cached_user_obj)
         check_in_budget(end_user_obj=return_obj)
         return return_obj
@@ -527,10 +526,7 @@ async def get_team_membership(
     # check if in cache
     cached_membership_obj = await user_api_key_cache.async_get_cache(key=_key)
     if cached_membership_obj is not None:
-        if isinstance(cached_membership_obj, dict):
-            return LiteLLM_TeamMembership(**cached_membership_obj)
-        elif isinstance(cached_membership_obj, LiteLLM_TeamMembership):
-            return cached_membership_obj
+        return LiteLLM_TeamMembership(**cached_membership_obj)
 
     # else, check db
     try:
@@ -542,8 +538,8 @@ async def get_team_membership(
         if response is None:
             return None
 
-        # save the team membership object to cache
-        await user_api_key_cache.async_set_cache(key=_key, value=response)
+        # save the team membership object to cache (store as dict)
+        await user_api_key_cache.async_set_cache(key=_key, value=response.dict())
 
         _response = LiteLLM_TeamMembership(**response.dict())
 
@@ -819,8 +815,9 @@ async def _cache_management_object(
     user_api_key_cache: DualCache,
     proxy_logging_obj: Optional[ProxyLogging],
 ):
+
     await user_api_key_cache.async_set_cache(
-        key=key, value=value, ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL
+        key=key, value=value, ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
     )
 
 

litellm/proxy/client/chat.py

@@ -1,5 +1,8 @@
+import json
+from typing import Any, Dict, Iterator, List, Optional, Union
+
 import requests
-from typing import List, Dict, Any, Optional, Union
+
 from .exceptions import UnauthorizedError
 
 
@@ -99,3 +102,91 @@ class ChatClient:
             if e.response.status_code == 401:
                 raise UnauthorizedError(e)
             raise
+
+    def completions_stream(
+        self,
+        model: str,
+        messages: List[Dict[str, str]],
+        temperature: Optional[float] = None,
+        top_p: Optional[float] = None,
+        n: Optional[int] = None,
+        max_tokens: Optional[int] = None,
+        presence_penalty: Optional[float] = None,
+        frequency_penalty: Optional[float] = None,
+        user: Optional[str] = None,
+    ) -> Iterator[Dict[str, Any]]:
+        """
+        Create a streaming chat completion.
+
+        Args:
+            model (str): The model to use for completion
+            messages (List[Dict[str, str]]): The messages to generate a completion for
+            temperature (Optional[float]): Sampling temperature between 0 and 2
+            top_p (Optional[float]): Nucleus sampling parameter between 0 and 1
+            n (Optional[int]): Number of completions to generate
+            max_tokens (Optional[int]): Maximum number of tokens to generate
+            presence_penalty (Optional[float]): Presence penalty between -2.0 and 2.0
+            frequency_penalty (Optional[float]): Frequency penalty between -2.0 and 2.0
+            user (Optional[str]): Unique identifier for the end user
+
+        Yields:
+            Dict[str, Any]: Streaming response chunks from the server
+
+        Raises:
+            UnauthorizedError: If the request fails with a 401 status code
+            requests.exceptions.RequestException: If the request fails with any other error
+        """
+        url = f"{self._base_url}/chat/completions"
+
+        # Build request data with required fields
+        data: Dict[str, Any] = {
+            "model": model, 
+            "messages": messages,
+            "stream": True
+        }
+
+        # Add optional parameters if provided
+        if temperature is not None:
+            data["temperature"] = temperature
+        if top_p is not None:
+            data["top_p"] = top_p
+        if n is not None:
+            data["n"] = n
+        if max_tokens is not None:
+            data["max_tokens"] = max_tokens
+        if presence_penalty is not None:
+            data["presence_penalty"] = presence_penalty
+        if frequency_penalty is not None:
+            data["frequency_penalty"] = frequency_penalty
+        if user is not None:
+            data["user"] = user
+
+        # Make streaming request
+        session = requests.Session()
+        try:
+            response = session.post(
+                url, 
+                headers=self._get_headers(), 
+                json=data, 
+                stream=True
+            )
+            response.raise_for_status()
+            
+            # Parse SSE stream
+            for line in response.iter_lines():
+                if line:
+                    line = line.decode('utf-8')
+                    if line.startswith('data: '):
+                        data_str = line[6:]  # Remove 'data: ' prefix
+                        if data_str.strip() == '[DONE]':
+                            break
+                        try:
+                            chunk = json.loads(data_str)
+                            yield chunk
+                        except json.JSONDecodeError:
+                            continue
+                            
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 401:
+                raise UnauthorizedError(e)
+            raise

litellm/proxy/client/cli/commands/auth.py

@@ -281,12 +281,12 @@ def prompt_team_selection_fallback(teams: List[Dict[str, Any]]) -> Optional[Dict
 
 def update_key_with_team(base_url: str, api_key: str, team_id: str) -> bool:
     """Update the API key to be associated with the selected team"""
-
+    from litellm.proxy._types import SpecialModelNames
     from litellm.proxy.client import Client
     
     client = Client(base_url=base_url, api_key=api_key)
     try:
-        result = client.keys.update(key=api_key, team_id=team_id)
+        client.keys.update(key=api_key, team_id=team_id, models=[SpecialModelNames.all_team_models.value])
         click.echo(f"✅ Successfully assigned key to team: {team_id}")
         return True
     except requests.exceptions.HTTPError as e:
@@ -301,6 +301,37 @@ def update_key_with_team(base_url: str, api_key: str, team_id: str) -> bool:
 
 # Polling-based authentication - no local server needed
 
+def _handle_team_assignment(base_url: str, api_key: str, user_id: str) -> None:
+    """Handle team fetching and assignment for the authenticated user."""
+    click.echo("\n" + "="*60)
+    click.echo("📋 Fetching your teams...")
+    
+    teams = get_user_teams(
+        base_url=base_url,
+        api_key=api_key,
+        user_id=user_id,
+    )
+    
+    if teams:
+        # Prompt for team selection (will display teams interactively)
+        selected_team = prompt_team_selection(teams)
+        
+        if selected_team:
+            team_id = selected_team.get('team_id')
+            if team_id:
+                click.echo(f"\n🔄 Assigning your key to team: {selected_team.get('team_alias', team_id)}")
+                success = update_key_with_team(base_url, api_key, team_id)
+                if success:
+                    click.echo(f"✅ Your CLI key is now associated with team: {selected_team.get('team_alias', team_id)}")
+                    click.echo(f"🎯 You can now access models: {', '.join(selected_team.get('models', ['All models']))}")
+                else:
+                    click.echo("⚠️ Key assignment failed, but you can still use the CLI")
+            else:
+                click.echo("ℹ️ Continuing without team assignment. You can assign a team later using the CLI.")
+    else:
+        click.echo("ℹ️ No teams found. You can create or join teams using the web interface.")
+
+
 @click.command(name="login")
 @click.pass_context
 def login(ctx: click.Context):
@@ -364,35 +395,8 @@ def login(ctx: click.Context):
                             click.echo(f"API Key: {api_key[:20]}...")
                             click.echo("You can now use the CLI without specifying --api-key")
                             
-                            # Fetch and display user's teams
-                            click.echo("\n" + "="*60)
-                            click.echo("📋 Fetching your teams...")
-                            
-                            teams = get_user_teams(
-                                base_url=base_url,
-                                api_key=api_key,
-                                user_id=data.get("user_id"),
-                            )
-                            
-                            
-                            if teams:
-                                # Prompt for team selection (will display teams interactively)
-                                selected_team = prompt_team_selection(teams)
-                                
-                                if selected_team:
-                                    team_id = selected_team.get('team_id')
-                                    if team_id:
-                                        click.echo(f"\n🔄 Assigning your key to team: {selected_team.get('team_alias', team_id)}")
-                                        success = update_key_with_team(base_url, api_key, team_id)
-                                        if success:
-                                            click.echo(f"✅ Your CLI key is now associated with team: {selected_team.get('team_alias', team_id)}")
-                                            click.echo(f"🎯 You can now access models: {', '.join(selected_team.get('models', ['All models']))}")
-                                        else:
-                                            click.echo("⚠️ Key assignment failed, but you can still use the CLI")
-                                else:
-                                    click.echo("ℹ️ Continuing without team assignment. You can assign a team later using the CLI.")
-                            else:
-                                click.echo("ℹ️ No teams found. You can create or join teams using the web interface.")
+                            # Handle team assignment
+                            _handle_team_assignment(base_url, api_key, data.get("user_id"))
                             
                             # Show available commands after successful login
                             click.echo("\n" + "="*60)

litellm/proxy/client/cli/commands/chat.py

@@ -1,42 +1,96 @@
 import json
-from typing import Optional
+import sys
+from typing import Any, Dict, List, Optional
 
 import click
-import rich
 import requests
+from rich.console import Console
+from rich.panel import Panel
+from rich.prompt import Prompt
+from rich.table import Table
 
+from ... import Client
 from ...chat import ChatClient
 
 
-@click.group()
-def chat():
-    """Chat with models through the LiteLLM proxy server"""
-    pass
+def _get_available_models(ctx: click.Context) -> List[Dict[str, Any]]:
+    """Get list of available models from the proxy server"""
+    try:
+        client = Client(base_url=ctx.obj["base_url"], api_key=ctx.obj["api_key"])
+        models_list = client.models.list()
+        # Ensure we return a list of dictionaries
+        if isinstance(models_list, list):
+            # Filter to ensure all items are dictionaries
+            return [model for model in models_list if isinstance(model, dict)]
+        return []
+    except Exception as e:
+        click.echo(f"Warning: Could not fetch models list: {e}", err=True)
+        return []
 
 
-@chat.command()
-@click.argument("model")
-@click.option(
-    "--message",
-    "-m",
-    multiple=True,
-    help="Messages in 'role:content' format (e.g. 'user:Hello'). Can be specified multiple times.",
-)
+def _select_model(console: Console, available_models: List[Dict[str, Any]]) -> Optional[str]:
+    """Interactive model selection"""
+    if not available_models:
+        console.print("[yellow]No models available or could not fetch models list.[/yellow]")
+        model_name = Prompt.ask("Please enter a model name")
+        return model_name if model_name.strip() else None
+    
+    # Display available models in a table
+    table = Table(title="Available Models")
+    table.add_column("Index", style="cyan", no_wrap=True)
+    table.add_column("Model ID", style="green")
+    table.add_column("Owned By", style="yellow")
+    MAX_MODELS_TO_DISPLAY = 200
+    
+    models_to_display: List[Dict[str, Any]] = available_models[:MAX_MODELS_TO_DISPLAY]
+    for i, model in enumerate(models_to_display):  # Limit to first 200 models
+        table.add_row(
+            str(i + 1),
+            str(model.get("id", "")),
+            str(model.get("owned_by", ""))
+        )
+    
+    if len(available_models) > MAX_MODELS_TO_DISPLAY:
+        console.print(f"\n[dim]... and {len(available_models) - MAX_MODELS_TO_DISPLAY} more models[/dim]")
+    
+    console.print(table)
+    
+    while True:
+        try:
+            choice = Prompt.ask(
+                "\nSelect a model by entering the index number (or type a model name directly)",
+                default="1"
+            ).strip()
+            
+            # Try to parse as index
+            try:
+                index = int(choice) - 1
+                if 0 <= index < len(available_models):
+                    return available_models[index]["id"]
+                else:
+                    console.print(f"[red]Invalid index. Please enter a number between 1 and {len(available_models)}[/red]")
+                    continue
+            except ValueError:
+                # Not a number, treat as model name
+                if choice:
+                    return choice
+                else:
+                    console.print("[red]Please enter a valid model name or index[/red]")
+                    continue
+                    
+        except KeyboardInterrupt:
+            console.print("\n[yellow]Model selection cancelled.[/yellow]")
+            return None
+
+
+@click.command()
+@click.argument("model", required=False)
 @click.option(
     "--temperature",
     "-t",
     type=float,
-    help="Sampling temperature between 0 and 2",
-)
-@click.option(
-    "--top-p",
-    type=float,
-    help="Nucleus sampling parameter between 0 and 1",
-)
-@click.option(
-    "--n",
-    type=int,
-    help="Number of completions to generate",
+    default=0.7,
+    help="Sampling temperature between 0 and 2 (default: 0.7)",
 )
 @click.option(
     "--max-tokens",
@@ -44,65 +98,271 @@ def chat():
     help="Maximum number of tokens to generate",
 )
 @click.option(
-    "--presence-penalty",
-    type=float,
-    help="Presence penalty between -2.0 and 2.0",
-)
-@click.option(
-    "--frequency-penalty",
-    type=float,
-    help="Frequency penalty between -2.0 and 2.0",
-)
-@click.option(
-    "--user",
+    "--system",
+    "-s",
     type=str,
-    help="Unique identifier for the end user",
+    help="System message to set the behavior of the assistant",
 )
 @click.pass_context
-def completions(
+def chat(
     ctx: click.Context,
-    model: str,
-    message: tuple[str, ...],
-    temperature: Optional[float] = None,
-    top_p: Optional[float] = None,
-    n: Optional[int] = None,
+    model: Optional[str],
+    temperature: float,
     max_tokens: Optional[int] = None,
-    presence_penalty: Optional[float] = None,
-    frequency_penalty: Optional[float] = None,
-    user: Optional[str] = None,
+    system: Optional[str] = None,
 ):
-    """Create a chat completion"""
-    if not message:
-        raise click.UsageError("At least one message is required")
-
-    # Parse messages from role:content format
-    messages = []
-    for msg in message:
-        try:
-            role, content = msg.split(":", 1)
-            messages.append({"role": role.strip(), "content": content.strip()})
-        except ValueError:
-            raise click.BadParameter(f"Invalid message format: {msg}. Expected format: 'role:content'")
-
+    """Interactive chat with streaming responses
+    
+    Examples:
+    
+        # Chat with a specific model
+        litellm-proxy chat gpt-4
+        
+        # Chat without specifying model (will show model selection)
+        litellm-proxy chat
+        
+        # Chat with custom settings
+        litellm-proxy chat gpt-4 --temperature 0.9 --system "You are a helpful coding assistant"
+    """
+    console = Console()
+    
+    # If no model specified, show model selection
+    if not model:
+        available_models = _get_available_models(ctx)
+        model = _select_model(console, available_models)
+        if not model:
+            console.print("[red]No model selected. Exiting.[/red]")
+            return
+    
     client = ChatClient(ctx.obj["base_url"], ctx.obj["api_key"])
+    
+    # Initialize conversation history
+    messages: List[Dict[str, Any]] = []
+    
+    # Add system message if provided
+    if system:
+        messages.append({"role": "system", "content": system})
+    
+    # Display welcome message
+    console.print(Panel.fit(
+        f"[bold blue]LiteLLM Interactive Chat[/bold blue]\n"
+        f"Model: [green]{model}[/green]\n"
+        f"Temperature: [yellow]{temperature}[/yellow]\n"
+        f"Max Tokens: [yellow]{max_tokens or 'unlimited'}[/yellow]\n\n"
+        f"Type your messages and press Enter. Type '/quit' or '/exit' to end the session.\n"
+        f"Type '/help' for more commands.",
+        title="🤖 Chat Session"
+    ))
+    
     try:
-        response = client.completions(
+        while True:
+            # Get user input
+            try:
+                user_input = console.input("\n[bold cyan]You:[/bold cyan] ").strip()
+            except (EOFError, KeyboardInterrupt):
+                console.print("\n[yellow]Chat session ended.[/yellow]")
+                break
+            
+            # Handle special commands
+            should_exit, messages, new_model = _handle_special_commands(
+                console, user_input, messages, system, ctx
+            )
+            
+            if should_exit:
+                break
+            if new_model:
+                model = new_model
+            
+            # Check if this was a special command that was handled (not a normal message)
+            if user_input.lower().startswith(('/quit', '/exit', '/q', '/help', '/clear', '/history', '/save', '/load', '/model')) or not user_input:
+                continue
+            
+            # Add user message to conversation
+            messages.append({"role": "user", "content": user_input})
+            
+            # Display assistant label
+            console.print("\n[bold green]Assistant:[/bold green]")
+            
+            # Stream the response
+            assistant_content = _stream_response(
+                console=console,
+                client=client,
+                model=model,
+                messages=messages,
+                temperature=temperature,
+                max_tokens=max_tokens,
+            )
+            
+            # Add assistant message to conversation history
+            if assistant_content:
+                messages.append({"role": "assistant", "content": assistant_content})
+            else:
+                console.print("[red]Error: No content received from the model[/red]")
+                    
+    except KeyboardInterrupt:
+        console.print("\n[yellow]Chat session interrupted.[/yellow]")
+
+
+def _show_help(console: Console):
+    """Show help for interactive chat commands"""
+    help_text = """
+[bold]Interactive Chat Commands:[/bold]
+
+[cyan]/help[/cyan]     - Show this help message
+[cyan]/quit[/cyan]     - Exit the chat session (also /exit, /q)
+[cyan]/clear[/cyan]    - Clear conversation history
+[cyan]/history[/cyan]  - Show conversation history
+[cyan]/model[/cyan]    - Switch to a different model
+[cyan]/save <name>[/cyan] - Save conversation to file
+[cyan]/load <name>[/cyan] - Load conversation from file
+
+[bold]Tips:[/bold]
+- Your conversation history is maintained during the session
+- Use Ctrl+C to interrupt at any time
+- Responses are streamed in real-time
+- You can switch models mid-conversation with /model
+    """
+    console.print(Panel(help_text, title="Help"))
+
+
+def _show_history(console: Console, messages: List[Dict[str, Any]]):
+    """Show conversation history"""
+    if not messages:
+        console.print("[yellow]No conversation history.[/yellow]")
+        return
+    
+    console.print(Panel.fit("[bold]Conversation History[/bold]", title="History"))
+    
+    for i, message in enumerate(messages, 1):
+        role = message["role"]
+        content = message["content"]
+        
+        if role == "system":
+            console.print(f"[dim]{i}. [bold magenta]System:[/bold magenta] {content}[/dim]")
+        elif role == "user":
+            console.print(f"{i}. [bold cyan]You:[/bold cyan] {content}")
+        elif role == "assistant":
+            console.print(f"{i}. [bold green]Assistant:[/bold green] {content[:100]}{'...' if len(content) > 100 else ''}")
+
+
+def _save_conversation(console: Console, messages: List[Dict[str, Any]], command: str):
+    """Save conversation to a file"""
+    parts = command.split()
+    if len(parts) < 2:
+        console.print("[red]Usage: /save <filename>[/red]")
+        return
+    
+    filename = parts[1]
+    if not filename.endswith('.json'):
+        filename += '.json'
+    
+    try:
+        with open(filename, 'w') as f:
+            json.dump(messages, f, indent=2)
+        console.print(f"[green]Conversation saved to {filename}[/green]")
+    except Exception as e:
+        console.print(f"[red]Error saving conversation: {e}[/red]")
+
+
+def _load_conversation(console: Console, command: str, system: Optional[str]) -> List[Dict[str, Any]]:
+    """Load conversation from a file"""
+    parts = command.split()
+    if len(parts) < 2:
+        console.print("[red]Usage: /load <filename>[/red]")
+        return []
+    
+    filename = parts[1]
+    if not filename.endswith('.json'):
+        filename += '.json'
+    
+    try:
+        with open(filename, 'r') as f:
+            messages = json.load(f)
+        console.print(f"[green]Conversation loaded from {filename}[/green]")
+        return messages
+    except FileNotFoundError:
+        console.print(f"[red]File not found: {filename}[/red]")
+    except Exception as e:
+        console.print(f"[red]Error loading conversation: {e}[/red]")
+    
+    # Return empty list or just system message if load failed
+    if system:
+        return [{"role": "system", "content": system}]
+    return []
+
+
+def _handle_special_commands(
+    console: Console, 
+    user_input: str, 
+    messages: List[Dict[str, Any]], 
+    system: Optional[str],
+    ctx: click.Context
+) -> tuple[bool, List[Dict[str, Any]], Optional[str]]:
+    """Handle special chat commands. Returns (should_exit, updated_messages, updated_model)"""
+    if user_input.lower() in ['/quit', '/exit', '/q']:
+        console.print("[yellow]Chat session ended.[/yellow]")
+        return True, messages, None
+    elif user_input.lower() == '/help':
+        _show_help(console)
+        return False, messages, None
+    elif user_input.lower() == '/clear':
+        new_messages = []
+        if system:
+            new_messages.append({"role": "system", "content": system})
+        console.print("[green]Conversation history cleared.[/green]")
+        return False, new_messages, None
+    elif user_input.lower() == '/history':
+        _show_history(console, messages)
+        return False, messages, None
+    elif user_input.lower().startswith('/save'):
+        _save_conversation(console, messages, user_input)
+        return False, messages, None
+    elif user_input.lower().startswith('/load'):
+        new_messages = _load_conversation(console, user_input, system)
+        return False, new_messages, None
+    elif user_input.lower() == '/model':
+        available_models = _get_available_models(ctx)
+        new_model = _select_model(console, available_models)
+        if new_model:
+            console.print(f"[green]Switched to model: {new_model}[/green]")
+            return False, messages, new_model
+        return False, messages, None
+    elif not user_input:
+        return False, messages, None
+    
+    # Not a special command
+    return False, messages, None
+
+
+def _stream_response(console: Console, client: ChatClient, model: str, messages: List[Dict[str, Any]], temperature: float, max_tokens: Optional[int]) -> Optional[str]:
+    """Stream the model response and return the complete content"""
+    try:
+        assistant_content = ""
+        for chunk in client.completions_stream(
             model=model,
             messages=messages,
             temperature=temperature,
-            top_p=top_p,
-            n=n,
             max_tokens=max_tokens,
-            presence_penalty=presence_penalty,
-            frequency_penalty=frequency_penalty,
-            user=user,
-        )
-        rich.print_json(data=response)
+        ):
+            if "choices" in chunk and len(chunk["choices"]) > 0:
+                delta = chunk["choices"][0].get("delta", {})
+                content = delta.get("content", "")
+                if content:
+                    assistant_content += content
+                    console.print(content, end="")
+                    sys.stdout.flush()
+        
+        console.print()  # Add newline after streaming
+        return assistant_content if assistant_content else None
+        
     except requests.exceptions.HTTPError as e:
-        click.echo(f"Error: HTTP {e.response.status_code}", err=True)
+        console.print(f"\n[red]Error: HTTP {e.response.status_code}[/red]")
         try:
             error_body = e.response.json()
-            rich.print_json(data=error_body)
+            console.print(f"[red]{error_body.get('error', {}).get('message', 'Unknown error')}[/red]")
         except json.JSONDecodeError:
-            click.echo(e.response.text, err=True)
-        raise click.Abort()
+            console.print(f"[red]{e.response.text}[/red]")
+        return None
+    except Exception as e:
+        console.print(f"\n[red]Error: {str(e)}[/red]")
+        return None

litellm/proxy/client/cli/commands/teams.py

@@ -80,11 +80,8 @@ def list(ctx: click.Context):
         display_teams_table(teams)
     except requests.exceptions.HTTPError as e:
         click.echo(f"Error: HTTP {e.response.status_code}", err=True)
-        try:
-            error_body = e.response.json()
-            click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True)
-        except:
-            click.echo(e.response.text, err=True)
+        error_body = e.response.json()
+        click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True)
         raise click.Abort()
     except Exception as e:
         click.echo(f"Error: {str(e)}", err=True)
@@ -107,12 +104,8 @@ def available(ctx: click.Context):
             click.echo("ℹ️ No available teams to join.")
     except requests.exceptions.HTTPError as e:
         click.echo(f"Error: HTTP {e.response.status_code}", err=True)
-        try:
-            error_body = e.response.json()
-            click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True)
-        except:
-            click.echo(e.response.text, err=True)
-        raise click.Abort()
+        error_body = e.response.json()
+        click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True)
     except Exception as e:
         click.echo(f"Error: {str(e)}", err=True)
         raise click.Abort()
@@ -152,7 +145,7 @@ def assign_key(ctx: click.Context, team_id: Optional[str]):
         # Update the key with the selected team
         if team_id:
             click.echo(f"\n🔄 Assigning your key to team: {team_id}")
-            result = client.keys.update(key=api_key, team_id=team_id)
+            client.keys.update(key=api_key, team_id=team_id)
             click.echo(f"✅ Successfully assigned key to team: {team_id}")
             
             # Show team details if available
@@ -168,11 +161,8 @@ def assign_key(ctx: click.Context, team_id: Optional[str]):
         
     except requests.exceptions.HTTPError as e:
         click.echo(f"Error: HTTP {e.response.status_code}", err=True)
-        try:
-            error_body = e.response.json()
-            click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True)
-        except:
-            click.echo(e.response.text, err=True)
+        error_body = e.response.json()
+        click.echo(f"Details: {error_body.get('detail', 'Unknown error')}", err=True)
         raise click.Abort()
     except Exception as e:
         click.echo(f"Error: {str(e)}", err=True)

litellm/proxy/client/cli/interface.py

@@ -84,7 +84,7 @@ def show_commands():
         ("whoami", "Show current authentication status"),
         ("models", "Manage and view model configurations"),
         ("credentials", "Manage API credentials"),
-        ("chat", "Interactive chat with models"),
+        ("chat", "Interactive streaming chat with models"),
         ("http", "Make HTTP requests to the proxy"),
         ("keys", "Manage API keys"),
         ("teams", "Manage teams and team assignments"),

litellm/proxy/client/client.py

@@ -1,5 +1,7 @@
 from typing import Optional
 
+from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
+
 from .chat import ChatClient
 from .credentials import CredentialsManagementClient
 from .http_client import HTTPClient
@@ -27,7 +29,7 @@ class Client:
             timeout: Request timeout in seconds (default: 30)
         """
         self._base_url = base_url.rstrip("/")  # Remove trailing slash if present
-        self._api_key = api_key
+        self._api_key = get_litellm_gateway_api_key() or api_key
 
         # Initialize resource clients
 

litellm/proxy/client/keys.py

@@ -274,14 +274,15 @@ class KeysManagementClient:
             data["aliases"] = aliases
         request = requests.Request("POST", url, headers=self._get_headers(), json=data)
         session = requests.Session()
+        response_text: Optional[str] = None
         try:
             response = session.send(request.prepare())
+            response_text = response.text
             response.raise_for_status()
             return response.json()
-        except requests.exceptions.HTTPError as e:
-            if e.response.status_code == 401:
-                raise UnauthorizedError(e)
-            raise
+        except Exception:
+            raise Exception(f"Error updating key: {response_text}")
+
 
     def info(self, key: str, return_request: bool = False) -> Union[Dict[str, Any], requests.Request]:
         """

litellm/proxy/client/teams.py

@@ -1,6 +1,6 @@
 """Teams management client for LiteLLM proxy."""
 
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 import requests
 
@@ -99,7 +99,7 @@ class TeamsManagementClient:
             UnauthorizedError: If authentication fails
         """
         url = f"{self._base_url}/v2/team/list"
-        params = {
+        params: Dict[str, Union[str, int]] = {
             "page": page,
             "page_size": page_size,
             "sort_order": sort_order,

litellm/proxy/common_request_processing.py

@@ -14,7 +14,6 @@ from typing import (
     Union,
 )
 
-import fastuuid as uuid
 import httpx
 import orjson
 from fastapi import HTTPException, Request, status
@@ -22,6 +21,7 @@ from fastapi.responses import Response, StreamingResponse
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm._uuid import uuid
 from litellm.constants import (
     DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE,
     STREAM_SSE_DATA_PREFIX,

litellm/proxy/common_utils/callback_utils.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Literal, Optional
 
 import litellm
 from litellm import get_secret
@@ -289,7 +289,7 @@ def initialize_callbacks_on_proxy(  # noqa: PLR0915
 
 def get_model_group_from_litellm_kwargs(kwargs: dict) -> Optional[str]:
     _litellm_params = kwargs.get("litellm_params", None) or {}
-    _metadata = _litellm_params.get("metadata", None) or {}
+    _metadata = _litellm_params.get(get_metadata_variable_name_from_kwargs(kwargs)) or {}
     _model_group = _metadata.get("model_group", None)
     if _model_group is not None:
         return _model_group
@@ -365,3 +365,20 @@ def add_guardrail_to_applied_guardrails_header(
         _metadata["applied_guardrails"].append(guardrail_name)
     else:
         _metadata["applied_guardrails"] = [guardrail_name]
+
+
+def get_metadata_variable_name_from_kwargs(
+        kwargs: dict
+    ) -> Literal["metadata", "litellm_metadata"]:
+        """
+        Helper to return what the "metadata" field should be called in the request data
+
+        - New endpoints return `litellm_metadata`
+        - Old endpoints return `metadata`
+
+        Context:
+        - LiteLLM used `metadata` as an internal field for storing metadata
+        - OpenAI then started using this field for their metadata
+        - LiteLLM is now moving to using `litellm_metadata` for our metadata
+        """
+        return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"

litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py

@@ -21,11 +21,11 @@ from litellm.integrations.custom_guardrail import (
     CustomGuardrail,
     log_guardrail_information,
 )
-from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
+from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.utils import (
@@ -88,11 +88,11 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
     def _create_sanitize_request(
         self, content: str, source: Literal["user_prompt", "model_response"]
     ) -> dict:
-        """Create request body for Model Armor API."""
+        """Create request body for Model Armor API with correct camelCase field names."""
         if source == "user_prompt":
-            return {"user_prompt_data": {"text": content}}
+            return {"userPromptData": {"text": content}}
         else:
-            return {"model_response_data": {"text": content}}
+            return {"modelResponseData": {"text": content}}
 
     def _extract_content_from_response(
         self, response: Union[Any, ModelResponse]
@@ -119,11 +119,16 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
 
     async def make_model_armor_request(
         self,
-        content: str,
-        source: Literal["user_prompt", "model_response"],
+        content: Optional[str] = None,
+        source: Literal["user_prompt", "model_response"] = "user_prompt",
         request_data: Optional[dict] = None,
+        file_bytes: Optional[bytes] = None,
+        file_type: Optional[str] = None,
     ) -> dict:
-        """Make request to Model Armor API."""
+        """
+        Make request to Model Armor API. Supports both text and file prompt sanitization.
+        If file_bytes and file_type are provided, file prompt sanitization is performed.
+        """
         # Get access token using VertexBase auth
         access_token, resolved_project_id = await self._ensure_access_token_async(
             credentials=self.credentials,
@@ -143,7 +148,14 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
             url = f"{endpoint}/v1/projects/{self.project_id}/locations/{self.location}/templates/{self.template_id}:sanitizeModelResponse"
 
         # Create request body
-        body = self._create_sanitize_request(content, source)
+        if file_bytes is not None and file_type is not None:
+            body = self.sanitize_file_prompt(file_bytes, file_type, source)
+        elif content is not None:
+            body = self._create_sanitize_request(content, source)
+        else:
+            raise ValueError(
+                "Either content or file_bytes and file_type must be provided."
+            )
 
         # Set headers
         headers = {
@@ -189,57 +201,112 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
             return await json_response
         return json_response
 
-    def _should_block_content(self, armor_response: dict) -> bool:
-        """Check if Model Armor response indicates content should be blocked."""
-        # Check the sanitizationResult from Model Armor API
+    def sanitize_file_prompt(
+        self, file_bytes: bytes, file_type: str, source: str = "user_prompt"
+    ) -> dict:
+        """
+        Helper to build the request body for file prompt sanitization for Model Armor.
+        file_type should be one of: PLAINTEXT_UTF8, PDF, WORD_DOCUMENT, EXCEL_DOCUMENT, POWERPOINT_DOCUMENT, TXT, CSV
+        Returns the request body dict.
+        """
+        import base64
+
+        base64_data = base64.b64encode(file_bytes).decode("utf-8")
+        if source == "user_prompt":
+            return {
+                "userPromptData": {
+                    "byteItem": {"byteDataType": file_type, "byteData": base64_data}
+                }
+            }
+        else:
+            return {
+                "modelResponseData": {
+                    "byteItem": {"byteDataType": file_type, "byteData": base64_data}
+                }
+            }
+
+    def _should_block_content(self, armor_response: dict, allow_sanitization: bool = False) -> bool:
+        """Check if Model Armor response indicates content should be blocked, including both inspectResult and deidentifyResult."""
         sanitization_result = armor_response.get("sanitizationResult", {})
         filter_results = sanitization_result.get("filterResults", {})
 
-        # Check blocking filters (these should cause the request to be blocked)
-        # RAI (Responsible AI) filters
-        rai_results = filter_results.get("rai", {}).get("raiFilterResult", {})
-        if rai_results.get("matchState") == "MATCH_FOUND":
-            return True
-
-        # Prompt injection and jailbreak filters
-        pi_jailbreak = filter_results.get("piAndJailbreakFilterResult", {})
-        if pi_jailbreak.get("matchState") == "MATCH_FOUND":
-            return True
-
-        # Malicious URI filters
-        malicious_uri = filter_results.get("maliciousUriFilterResult", {})
-        if malicious_uri.get("matchState") == "MATCH_FOUND":
-            return True
-
-        # CSAM filters
-        csam = filter_results.get("csamFilterFilterResult", {})
-        if csam.get("matchState") == "MATCH_FOUND":
-            return True
-
-        # Virus scan filters
-        virus_scan = filter_results.get("virusScanFilterResult", {})
-        if virus_scan.get("matchState") == "MATCH_FOUND":
-            return True
+        # filterResults can be a dict (named keys) or a list (array of filter result dicts)
+        filter_result_items = []
+        if isinstance(filter_results, dict):
+            filter_result_items = list(filter_results.values())
+        elif isinstance(filter_results, list):
+            filter_result_items = filter_results
 
+        for filt in filter_result_items:
+            # Check RAI, PI/Jailbreak, Malicious URI, CSAM, Virus scan as before
+            if filt.get("raiFilterResult", {}).get("matchState") == "MATCH_FOUND":
+                return True
+            if (
+                filt.get("piAndJailbreakFilterResult", {}).get("matchState")
+                == "MATCH_FOUND"
+            ):
+                return True
+            if (
+                filt.get("maliciousUriFilterResult", {}).get("matchState")
+                == "MATCH_FOUND"
+            ):
+                return True
+            if (
+                filt.get("csamFilterFilterResult", {}).get("matchState")
+                == "MATCH_FOUND"
+            ):
+                return True
+            if filt.get("virusScanFilterResult", {}).get("matchState") == "MATCH_FOUND":
+                return True
+            # Check sdpFilterResult for both inspectResult and deidentifyResult
+            sdp = filt.get("sdpFilterResult")
+            if sdp:
+                if sdp.get("inspectResult", {}).get("matchState") == "MATCH_FOUND":
+                    return True
+                # Only block on deidentifyResult if sanitization is not allowed
+                if sdp.get("deidentifyResult", {}).get("matchState") == "MATCH_FOUND":
+                    if not allow_sanitization:
+                        return True
+        # Fallback dict code removed; all cases handled above
         return False
 
     def _get_sanitized_content(self, armor_response: dict) -> Optional[str]:
-        """Extract sanitized content from Model Armor response."""
-        # Model Armor returns sanitized content in the sanitizationResult
-        sanitization_result = armor_response.get("sanitizationResult", {})
+        """
+        Get the sanitized content from a Model Armor response, if available.
+        Looks for sanitized text in deidentifyResult, and falls back to root-level fields if not found.
+        """
+        result = armor_response.get("sanitizationResult", {})
+        filter_results = result.get("filterResults", {})
 
-        # Check for sdp structure (for deidentification)
-        filter_results = sanitization_result.get("filterResults", {})
-        sdp = filter_results.get("sdp", {}).get("sdpFilterResult")
+        # filterResults can be a dict (single filter) or a list (multiple filters)
+        filters = (
+            list(filter_results.values())
+            if isinstance(filter_results, dict)
+            else filter_results
+            if isinstance(filter_results, list)
+            else []
+        )
 
-        if sdp is not None:
-            # Model Armor returns sanitized text under deidentifyResult in sdp
-            deidentify_result = sdp.get("deidentifyResult", {})
-            sanitized_text = deidentify_result.get("data", {}).get("text", "")
-            if deidentify_result.get("matchState") == "MATCH_FOUND" and sanitized_text:
-                return sanitized_text
+        # Prefer sanitized text from deidentifyResult if present
+        for filter_entry in filters:
+            sdp = filter_entry.get("sdpFilterResult")
+            if sdp:
+                deid = sdp.get("deidentifyResult", {})
+                sanitized = deid.get("data", {}).get("text", "")
+                # If Model Armor found something and returned a sanitized version, use it
+                if deid.get("matchState") == "MATCH_FOUND" and sanitized:
+                    return sanitized
 
-        # Fallback to checking root level
+        # If no deidentifyResult, optionally check for inspectResult (rare, but could have findings)
+        for filter_entry in filters:
+            sdp = filter_entry.get("sdpFilterResult")
+            if sdp:
+                inspect = sdp.get("inspectResult", {})
+                # If Model Armor flagged something but didn't sanitize, return None
+                if inspect.get("matchState") == "MATCH_FOUND":
+                    return None
+
+        # Fallback: if Model Armor put sanitized text at the root, use it
         return armor_response.get("sanitizedText") or armor_response.get("text")
 
     def _process_response(
@@ -344,11 +411,11 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
                 #   fail_on_error=False) we still want the correct status reflected.
                 metadata["_model_armor_status"] = (
                     "blocked"
-                    if self._should_block_content(armor_response)
+                    if self._should_block_content(armor_response, allow_sanitization=self.mask_request_content)
                     else "success"
                 )
             # Check if content should be blocked
-            if self._should_block_content(armor_response):
+            if self._should_block_content(armor_response, allow_sanitization=self.mask_request_content):
                 raise HTTPException(
                     status_code=400,
                     detail={
@@ -429,12 +496,12 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
                 metadata["_model_armor_response"] = armor_response
                 metadata["_model_armor_status"] = (
                     "blocked"
-                    if self._should_block_content(armor_response)
+                    if self._should_block_content(armor_response, allow_sanitization=self.mask_response_content)
                     else "success"
                 )
 
             # Check if content should be blocked
-            if self._should_block_content(armor_response):
+            if self._should_block_content(armor_response, allow_sanitization=self.mask_response_content):
                 raise HTTPException(
                     status_code=400,
                     detail={

litellm/proxy/hooks/dynamic_rate_limiter_v3.py

@@ -40,7 +40,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
 
     def _get_priority_weight(self, priority: Optional[str]) -> float:
         """Get the weight for a given priority from litellm.priority_reservation"""
-        weight: float = 1.0
+        weight: float = litellm.priority_reservation_settings.default_priority
         if (
             litellm.priority_reservation is None
             or priority not in litellm.priority_reservation

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -25,6 +25,7 @@ from litellm import DualCache
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
@@ -708,6 +709,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         )
         from litellm.proxy.common_utils.callback_utils import (
             get_model_group_from_litellm_kwargs,
+            get_metadata_variable_name_from_kwargs
         )
         from litellm.types.caching import RedisPipelineIncrementOperation
         from litellm.types.utils import ModelResponse, Usage
@@ -723,7 +725,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             )
 
             # Get metadata from kwargs
-            litellm_metadata = kwargs["litellm_params"]["metadata"]
+            litellm_metadata = kwargs["litellm_params"].get(get_metadata_variable_name_from_kwargs(kwargs), {})
             if litellm_metadata is None:
                 return
             user_api_key = litellm_metadata.get("user_api_key")
@@ -736,7 +738,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
 
             # Get total tokens from response
             total_tokens = 0
-            if isinstance(response_obj, ModelResponse):
+            # spot fix for /responses api
+            if (isinstance(response_obj, ModelResponse) or isinstance(response_obj, BaseLiteLLMOpenAIResponseObject)):
                 _usage = getattr(response_obj, "usage", None)
                 if _usage and isinstance(_usage, Usage):
                     if rate_limit_type == "output":

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -1232,13 +1232,11 @@ def validate_key_team_change(
             )
 
     # Check if the key's user_id is a member of the team
+    member_object = _get_user_in_team(
+        team_table=cast(LiteLLM_TeamTableCachedObj, team), user_id=key.user_id
+    )
     if key.user_id is not None:
-        is_member = False
-        for member in team.members_with_roles:
-            if member.user_id == key.user_id:
-                is_member = True
-                break
-        if not is_member:
+        if not member_object:
             raise HTTPException(
                 status_code=403,
                 detail=f"User={key.user_id} is not a member of the team={team.team_id}. Check team members via `/team/info`.",
@@ -1265,10 +1263,17 @@ def validate_key_team_change(
         team_obj=team,
     ):
         return
+    # this teams member permissions allow updating a
+    elif TeamMemberPermissionChecks.does_team_member_have_permissions_for_endpoint(
+        team_member_object=member_object,
+        team_table=cast(LiteLLM_TeamTableCachedObj, team),
+        route=KeyManagementRoutes.KEY_UPDATE.value,
+    ):
+        return
     else:
         raise HTTPException(
             status_code=403,
-            detail=f"User={change_initiated_by.user_id} is not a Proxy Admin or Team Admin for team={team.team_id}.",
+            detail=f"User={change_initiated_by.user_id} is not a Proxy Admin or Team Admin for team={team.team_id}. Please ask your Proxy Admin to allow this action under 'Member Permissions' for this team.",
         )
 
 

litellm/proxy/management_endpoints/ui_sso.py

@@ -731,7 +731,6 @@ async def _create_new_cli_key(
         config={},
         spend=0,
         user_id=user_id,
-        team_id="litellm-cli",
         table_name="key",
         token=key,
     )
@@ -810,7 +809,7 @@ async def cli_poll_key(key_id: str):
         key_obj = await prisma_client.db.litellm_verificationtoken.find_unique(
             where={"token": hashed_token}
         )
-        key_obj: LiteLLM_VerificationToken = cast(LiteLLM_VerificationToken, key_obj)
+        key_obj = cast(LiteLLM_VerificationToken, key_obj)
 
         if key_obj:
             verbose_proxy_logger.info(f"CLI key found: {key_id}")
@@ -1503,7 +1502,7 @@ class SSOAuthenticationHandler:
         ## CHECK IF ROLE ALLOWED TO USE PROXY ##
         is_admin_only_access = check_is_admin_only_access(ui_access_mode or {})
         if is_admin_only_access:
-            has_access = has_admin_ui_access(user_role)
+            has_access = has_admin_ui_access(user_role or "")
             if not has_access:
                 raise HTTPException(
                     status_code=401,
@@ -1547,7 +1546,7 @@ class SSOAuthenticationHandler:
             user_id=cast(str, user_id),
             key=key,
             user_email=user_email,
-            user_role=user_role,
+            user_role=user_role or LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value,
             login_method="sso",
             premium_user=premium_user,
             auth_header_name=general_settings.get(

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -459,6 +459,14 @@ async def anthropic_proxy_route(
         region_name=None,
     )
 
+    custom_headers = {}
+    if (
+        "authorization" not in request.headers
+        and "x-api-key" not in request.headers
+        and anthropic_api_key is not None
+    ):
+        custom_headers["x-api-key"] = "{}".format(anthropic_api_key)
+
     ## check for streaming
     is_streaming_request = await is_streaming_request_fn(request)
 
@@ -466,7 +474,7 @@ async def anthropic_proxy_route(
     endpoint_func = create_pass_through_route(
         endpoint=endpoint,
         target=str(updated_url),
-        custom_headers={"x-api-key": "{}".format(anthropic_api_key)},
+        custom_headers=custom_headers,
         _forward_headers=True,
     )  # dynamically construct pass-through endpoint based on incoming path
     received_value = await endpoint_func(

litellm/proxy/proxy_server.py

@@ -35,6 +35,7 @@ from litellm.constants import (
     LITELLM_SETTINGS_SAFE_DB_OVERRIDES,
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.utils import load_credentials_from_list
 from litellm.types.utils import (
     ModelResponse,
     ModelResponseStream,
@@ -5894,6 +5895,7 @@ async def token_counter(request: TokenCountRequest, call_endpoint: bool = False)
             pass
     if deployment is not None:
         litellm_model_name = deployment.get("litellm_params", {}).get("model")
+        load_credentials_from_list(deployment.get("litellm_params", {}))
         # remove the custom_llm_provider_prefix in the litellm_model_name
         if "/" in litellm_model_name:
             litellm_model_name = litellm_model_name.split("/", 1)[1]

litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py

@@ -493,21 +493,32 @@ async def update_sso_settings(sso_config: SSOConfig):
         config["general_settings"] = {}
 
     # Update environment variables in config and in memory
-    sso_data = sso_config.model_dump(exclude_none=True)
+    sso_data = sso_config.model_dump()
     for field_name, value in sso_data.items():
-
-        if field_name == "user_email" and value is not None:
-            # Store user_email in general_settings instead of environment variables
-            config["general_settings"]["proxy_admin_email"] = value
-        elif field_name == "ui_access_mode" and value is not None:
-
-            config["general_settings"]["ui_access_mode"] = value
-        elif field_name in env_var_mapping and value is not None:
+        if field_name == "user_email":
+            if value:
+                # Store user_email in general_settings instead of environment variables
+                config["general_settings"]["proxy_admin_email"] = value
+            else:
+                # Clear user_email if null/empty
+                config["general_settings"].pop("proxy_admin_email", None)
+        elif field_name == "ui_access_mode":
+            if value:
+                config["general_settings"]["ui_access_mode"] = value
+            else:
+                # Clear ui_access_mode if null/empty
+                config["general_settings"].pop("ui_access_mode", None)
+        elif field_name in env_var_mapping and value:
             env_var_name = env_var_mapping[field_name]
             # Update in config
             config["environment_variables"][env_var_name] = value
             # Update in runtime environment
             os.environ[env_var_name] = value
+        elif field_name in env_var_mapping:
+            # Clear environment variable if value is null/empty
+            env_var_name = env_var_mapping[field_name]
+            config["environment_variables"].pop(env_var_name, None)
+            os.environ.pop(env_var_name, None)
 
     stored_config = config
     if len(config["environment_variables"]) > 0:

litellm/types/integrations/langfuse.py

@@ -7,3 +7,10 @@ class LangfuseLoggingConfig(TypedDict):
     langfuse_secret: Optional[str]
     langfuse_public_key: Optional[str]
     langfuse_host: Optional[str]
+
+
+class LangfuseUsageDetails(TypedDict):
+    input: Optional[int]
+    output: Optional[int]
+    cache_creation_input_tokens: Optional[int]
+    cache_read_input_tokens: Optional[int]

litellm/types/utils.py

@@ -13,7 +13,6 @@ from typing import (
     Union,
 )
 
-import fastuuid as uuid
 from aiohttp import FormData
 from openai._models import BaseModel as OpenAIObject
 from openai.types.audio.transcription_create_params import FileTypes  # type: ignore
@@ -33,6 +32,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator
 from typing_extensions import Callable, Dict, Required, TypedDict, override
 
 import litellm
+from litellm._uuid import uuid
 from litellm.types.llms.base import (
     BaseLiteLLMOpenAIResponseObject,
     LiteLLMPydanticObjectBase,
@@ -2633,3 +2633,18 @@ CostResponseTypes = Union[
     ImageResponse,
     TranscriptionResponse,
 ]
+
+
+class PriorityReservationSettings(BaseModel):
+    """
+    Settings for priority-based rate limiting reservation.
+    
+    Defines what priority to assign to keys without explicit priority metadata.
+    The priority_reservation mapping is configured separately via litellm.priority_reservation.
+    """
+    default_priority: float = Field(
+        default=0.5,
+        description="Priority level to assign to API keys without explicit priority metadata. Should match a key in litellm.priority_reservation."
+    )
+
+    model_config = ConfigDict(protected_namespaces=())

litellm/utils.py

@@ -40,7 +40,6 @@ from os.path import abspath, dirname, join
 
 import aiohttp
 import dotenv
-import fastuuid as uuid
 import httpx
 import openai
 import tiktoken
@@ -59,6 +58,7 @@ import litellm.litellm_core_utils.audio_utils.utils
 import litellm.litellm_core_utils.json_validation_rule
 import litellm.llms
 import litellm.llms.gemini
+from litellm._uuid import uuid
 from litellm.caching._internal_lru_cache import lru_cache_wrapper
 from litellm.caching.caching import DualCache
 from litellm.caching.caching_handler import CachingHandlerResponse, LLMCachingHandler

model_prices_and_context_window.json

@@ -560,10 +560,14 @@
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -824,10 +828,14 @@
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -4817,10 +4825,14 @@
         "cache_creation_input_token_cost_above_1hr": 6e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "anthropic",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -7264,6 +7276,18 @@
         "supports_prompt_caching": true,
         "supports_tool_choice": true
     },
+    "deepseek.v3-v1:0": {
+        "input_cost_per_token": 5.8e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 163840,
+        "max_output_tokens": 81920,
+        "max_tokens": 163840,
+        "mode": "chat",
+        "output_cost_per_token": 1.68e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },    
     "dolphin": {
         "input_cost_per_token": 5e-07,
         "litellm_provider": "nlp_cloud",
@@ -7620,10 +7644,14 @@
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -16260,10 +16288,12 @@
     "openrouter/anthropic/claude-sonnet-4": {
         "input_cost_per_image": 0.0048,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
         "litellm_provider": "openrouter",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "supports_assistant_prefill": true,
@@ -17605,6 +17635,54 @@
         "mode": "chat",
         "output_cost_per_token": 2.8e-07
     },
+    "qwen.qwen3-coder-480b-a35b-v1:0": {
+        "input_cost_per_token": 2.2e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262000,
+        "max_output_tokens": 65536,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 1.8e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "qwen.qwen3-235b-a22b-2507-v1:0": {
+        "input_cost_per_token": 2.2e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 131072,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 8.8e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "qwen.qwen3-coder-30b-a3b-v1:0": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 131072,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 6.0e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    }, 
+    "qwen.qwen3-32b-v1:0": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 16384,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 6.0e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },           
     "recraft/recraftv2": {
         "litellm_provider": "recraft",
         "mode": "image_generation",
@@ -19123,10 +19201,14 @@
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -20459,10 +20541,14 @@
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "vertex_ai-anthropic_models",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -20485,10 +20571,14 @@
         "cache_creation_input_token_cost": 3.75e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
+        "input_cost_per_token_above_200k_tokens": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 2.25e-05,
+        "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "vertex_ai-anthropic_models",
-        "max_input_tokens": 200000,
-        "max_output_tokens": 64000,
-        "max_tokens": 64000,
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "search_context_cost_per_query": {
@@ -21334,6 +21424,30 @@
             "/v1/audio/transcriptions"
         ]
     },
+    "vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "vertex_ai-qwen_models",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": {
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "vertex_ai-qwen_models",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
     "xai/grok-2": {
         "input_cost_per_token": 2e-06,
         "litellm_provider": "xai",
@@ -21586,6 +21700,35 @@
         "supports_tool_choice": true,
         "supports_web_search": true
     },
+    "xai/grok-4-fast-reasoning": {
+        "litellm_provider": "xai",
+        "max_input_tokens": 2e6,
+        "max_output_tokens": 2e6,
+        "max_tokens": 2e6,
+        "mode": "chat",
+        "input_cost_per_token": 0.2e-06,
+        "output_cost_per_token": 0.5e-06,
+        "cache_read_input_token_cost": 0.05e-06,
+        "source": "https://docs.x.ai/docs/models",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_web_search": true
+    },
+    "xai/grok-4-fast-non-reasoning": {
+        "litellm_provider": "xai",
+        "max_input_tokens": 2e6,
+        "max_output_tokens": 2e6,
+        "cache_read_input_token_cost": 0.05e-06,
+        "max_tokens": 2e6,
+        "mode": "chat",
+        "input_cost_per_token": 0.2e-06,
+        "output_cost_per_token": 0.5e-06,
+        "source": "https://docs.x.ai/docs/models",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_web_search": true
+    },
     "xai/grok-4-0709": {
         "input_cost_per_token": 3e-06,
         "litellm_provider": "xai",

poetry.lock

@@ -555,7 +555,7 @@ description = "Foreign Function Interface for Python calling C code."
 optional = false
 python-versions = ">=3.8"
 groups = ["main", "dev", "proxy-dev"]
-markers = "python_version < \"3.14\" and platform_python_implementation != \"PyPy\""
+markers = "python_version < \"3.10\" and platform_python_implementation != \"PyPy\""
 files = [
     {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"},
     {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"},
@@ -636,7 +636,7 @@ description = "Foreign Function Interface for Python calling C code."
 optional = false
 python-versions = ">=3.9"
 groups = ["main", "dev", "proxy-dev"]
-markers = "platform_python_implementation != \"PyPy\" and python_version >= \"3.14\""
+markers = "python_version >= \"3.10\" and platform_python_implementation != \"PyPy\""
 files = [
     {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"},
     {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"},
@@ -1339,9 +1339,10 @@ zstandard = ["zstandard"]
 name = "fastuuid"
 version = "0.12.0"
 description = "Python bindings to Rust's UUID library."
-optional = false
+optional = true
 python-versions = ">=3.8"
 groups = ["main"]
+markers = "extra == \"proxy\""
 files = [
     {file = "fastuuid-0.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:22a900ef0956aacf862b460e20541fdae2d7c340594fe1bd6fdcb10d5f0791a9"},
     {file = "fastuuid-0.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0302f5acf54dc75de30103025c5a95db06d6c2be36829043a0aa16fc170076bc"},
@@ -4361,7 +4362,7 @@ description = "C parser in Python"
 optional = false
 python-versions = ">=3.8"
 groups = ["main", "dev", "proxy-dev"]
-markers = "platform_python_implementation != \"PyPy\" and (python_version < \"3.14\" or implementation_name != \"PyPy\")"
+markers = "platform_python_implementation != \"PyPy\" and (implementation_name != \"PyPy\" or python_version < \"3.10\")"
 files = [
     {file = "pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934"},
     {file = "pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2"},
@@ -6746,11 +6747,11 @@ type = ["pytest-mypy"]
 caching = ["diskcache"]
 extra-proxy = ["azure-identity", "azure-keyvault-secrets", "google-cloud-iam", "google-cloud-kms", "prisma", "redisvl", "resend"]
 mlflow = ["mlflow"]
-proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "python-multipart", "pyyaml", "rich", "rq", "uvicorn", "uvloop", "websockets"]
+proxy = ["PyJWT", "apscheduler", "azure-identity", "azure-storage-blob", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "fastuuid", "gunicorn", "litellm-enterprise", "litellm-proxy-extras", "mcp", "orjson", "polars", "pynacl", "python-multipart", "pyyaml", "rich", "rq", "uvicorn", "uvloop", "websockets"]
 semantic-router = ["semantic-router"]
 utils = ["numpydoc"]
 
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.8.1,<4.0, !=3.9.7"
-content-hash = "dcec654bc4b233d2f0d160341d8adf1ba2017ccb74c104bff9ba4cf027ba0186"
+content-hash = "75004c6a23b70be86622fa417fd0d62fa3843e6e61c8dff8507ae5c967b7205d"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "1.77.3"
+version = "1.77.4"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT"
@@ -20,7 +20,6 @@ Documentation = "https://docs.litellm.ai"
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<4.0, !=3.9.7"
-fastuuid = ">=0.12.0"
 httpx = ">=0.23.0"
 openai = ">=1.99.5"
 python-dotenv = ">=0.2.0"
@@ -34,6 +33,7 @@ pydantic = "^2.5.0"
 jsonschema = "^4.22.0"
 pondpond = "^1.4.1"
 numpydoc = {version = "*", optional = true} # used in utils.py
+fastuuid = {version = ">=0.12.0", optional = true}
 
 uvicorn = {version = "^0.29.0", optional = true}
 uvloop = {version = "^0.21.0", optional = true, markers="sys_platform != 'win32'"}
@@ -93,6 +93,7 @@ proxy = [
     "litellm-enterprise",
     "rich",
     "polars",
+    "fastuuid",
 ]
 
 extra_proxy = [
@@ -115,6 +116,7 @@ semantic-router = ["semantic-router"]
 
 mlflow = ["mlflow"]
 
+
 [tool.isort]
 profile = "black"
 
@@ -157,7 +159,7 @@ requires = ["poetry-core", "wheel"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "1.77.3"
+version = "1.77.4"
 version_files = [
     "pyproject.toml:^version"
 ]

security.md

@@ -12,11 +12,6 @@
 - For installation and configuration, see: [Self-hosting guided](https://docs.litellm.ai/docs/proxy/deploy)
 - **Telemetry** We run no telemetry when you self host LiteLLM
 
-
-:::info
-✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
-:::
-
 ### LiteLLM Cloud
 
 - We encrypt all data stored using your `LITELLM_MASTER_KEY` and in transit using TLS.

tests/code_coverage_tests/check_fastuuid_usage.py

@@ -0,0 +1,87 @@
+import ast
+import os
+from typing import List, Dict, Any
+
+
+ALLOWED_FILE = os.path.normpath("litellm/_uuid.py")
+
+
+def _to_module_path(relative_path: str) -> str:
+    module = os.path.splitext(relative_path)[0].replace(os.sep, ".")
+    if module.endswith(".__init__"):
+        return module[: -len(".__init__")]
+    return module
+
+
+def _find_fastuuid_imports_in_file(
+    file_path: str, base_dir: str
+) -> List[Dict[str, Any]]:
+    results: List[Dict[str, Any]] = []
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            source = f.read()
+        tree = ast.parse(source, filename=file_path)
+    except Exception:
+        return results
+
+    relative = os.path.normpath(os.path.relpath(file_path, base_dir))
+    if relative == ALLOWED_FILE:
+        return results
+
+    module = _to_module_path(relative)
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                if alias.name == "fastuuid":
+                    results.append(
+                        {
+                            "file": relative,
+                            "line": getattr(node, "lineno", 0),
+                            "import": f"import {alias.name}",
+                            "module": module,
+                        }
+                    )
+        elif isinstance(node, ast.ImportFrom) and node.module == "fastuuid":
+            names = ", ".join([a.name for a in node.names])
+            results.append(
+                {
+                    "file": relative,
+                    "line": getattr(node, "lineno", 0),
+                    "import": f"from fastuuid import {names}",
+                    "module": module,
+                }
+            )
+
+    return results
+
+
+def scan_directory_for_fastuuid(base_dir: str) -> List[Dict[str, Any]]:
+    violations: List[Dict[str, Any]] = []
+    scan_root = os.path.join(base_dir, "litellm")
+    for root, _, files in os.walk(scan_root):
+        for filename in files:
+            if filename.endswith(".py"):
+                file_path = os.path.join(root, filename)
+                violations.extend(_find_fastuuid_imports_in_file(file_path, base_dir))
+    return violations
+
+
+def main() -> None:
+    base_dir = "."  # tests run from repo root in CI
+    violations = scan_directory_for_fastuuid(base_dir)
+    if violations:
+        print(
+            "\n🚨 fastuuid must only be imported inside litellm/_uuid.py. Found violations:"
+        )
+        for v in violations:
+            print(f"* {v['module']} ({v['file']}:{v['line']}) -> {v['import']}")
+        print("\n")
+        raise Exception(
+            "Found fastuuid imports outside litellm/_uuid.py. Use litellm._uuid.uuid or litellm._uuid.uuid4 instead."
+        )
+    else:
+        print("✅ No invalid fastuuid imports found.")
+
+
+if __name__ == "__main__":
+    main()

tests/guardrails_tests/test_model_armor_file_sanitization.py

@@ -1,75 +0,0 @@
-import sys
-import os
-import pytest
-from unittest.mock import AsyncMock
-from fastapi import HTTPException
-
-sys.path.insert(0, os.path.abspath("../.."))
-
-from litellm.proxy.guardrails.guardrail_hooks.model_armor.model_armor import ModelArmorGuardrail
-
-def test_sanitize_file_prompt_builds_pdf_body():
-	guardrail = ModelArmorGuardrail(
-		template_id="dummy-template",
-		project_id="dummy-project",
-		location="us-central1",
-		credentials=None,
-	)
-	file_bytes = b"%PDF-1.4 some pdf content"
-	file_type = "PDF"
-	body = guardrail.sanitize_file_prompt(file_bytes, file_type, source="user_prompt")
-	assert "userPromptData" in body
-	assert body["userPromptData"]["byteItem"]["byteDataType"] == "PDF"
-	import base64
-	assert body["userPromptData"]["byteItem"]["byteData"] == base64.b64encode(file_bytes).decode("utf-8")
-
-@pytest.mark.asyncio
-async def test_make_model_armor_request_file_prompt():
-	guardrail = ModelArmorGuardrail(
-		template_id="dummy-template",
-		project_id="dummy-project",
-		location="us-central1",
-		credentials=None,
-	)
-	file_bytes = b"My SSN is 123-45-6789."
-	file_type = "PLAINTEXT_UTF8"
-	armor_response = {
-		"sanitizationResult": {
-			"filterResults": [
-				{
-					"sdpFilterResult": {
-						"inspectResult": {
-							"executionState": "EXECUTION_SUCCESS",
-							"matchState": "MATCH_FOUND",
-							"findings": [
-								{"infoType": "US_SOCIAL_SECURITY_NUMBER", "likelihood": "LIKELY"}
-							]
-						},
-						"deidentifyResult": {
-							"executionState": "EXECUTION_SUCCESS",
-							"matchState": "MATCH_FOUND",
-							"data": {"text": "My SSN is [REDACTED]."}
-						}
-					}
-				}
-			]
-		}
-	}
-	class MockResponse:
-		def __init__(self, status_code, text, json_data):
-			self.status_code = status_code
-			self.text = text
-			self._json = json_data
-		def json(self):
-			return self._json
-	class MockHandler:
-		async def post(self, url, json, headers):
-			return MockResponse(200, str(armor_response), armor_response)
-	guardrail.async_handler = MockHandler()
-	guardrail._ensure_access_token_async = AsyncMock(return_value=("dummy-token", "dummy-project"))
-	result = await guardrail.make_model_armor_request(
-		file_bytes=file_bytes,
-		file_type=file_type,
-		source="user_prompt"
-	)
-	assert result["sanitizationResult"]["filterResults"][0]["sdpFilterResult"]["deidentifyResult"]["data"]["text"] == "My SSN is [REDACTED]."

tests/guardrails_tests/test_model_armor_guardrail.py

@@ -1,99 +0,0 @@
-import sys
-import os
-import pytest
-from unittest.mock import AsyncMock, patch
-from fastapi import HTTPException
-
-sys.path.insert(0, os.path.abspath("../.."))
-
-from litellm.proxy.guardrails.guardrail_hooks.model_armor.model_armor import ModelArmorGuardrail
-from litellm.proxy._types import UserAPIKeyAuth
-from litellm.caching.caching import DualCache
-
-@pytest.mark.asyncio
-async def test_model_armor_pre_call_hook_inspect_and_deidentify():
-    """
-    Test Model Armor guardrail pre-call hook for both inspectResult and deidentifyResult handling.
-    """
-    guardrail = ModelArmorGuardrail(
-        template_id="dummy-template",
-        project_id="dummy-project",
-        location="us-central1",
-        credentials=None,
-    )
-    armor_response = {
-        "sanitizationResult": {
-            "filterResults": [
-                {
-                    "sdpFilterResult": {
-                        "inspectResult": {
-                            "executionState": "EXECUTION_SUCCESS",
-                            "matchState": "NO_MATCH_FOUND",
-                            "findings": []
-                        },
-                        "deidentifyResult": {
-                            "executionState": "EXECUTION_SUCCESS",
-                            "matchState": "MATCH_FOUND",
-                            "data": {"text": "sanitized text here"}
-                        }
-                    }
-                }
-            ]
-        }
-    }
-    with patch.object(guardrail, "make_model_armor_request", AsyncMock(return_value=armor_response)):
-        user_api_key_dict = UserAPIKeyAuth(api_key="test_key")
-        cache = DualCache()
-        data = {
-            "messages": [
-                {"role": "system", "content": "You are a helpful assistant."},
-                {"role": "user", "content": "My SSN is 123-45-6789."}
-            ],
-            "model": "gpt-3.5-turbo",
-            "metadata": {}
-        }
-        guardrail.mask_request_content = True
-        with pytest.raises(HTTPException) as exc_info:
-            await guardrail.async_pre_call_hook(
-                user_api_key_dict=user_api_key_dict,
-                cache=cache,
-                data=data,
-                call_type="completion"
-            )
-        assert exc_info.value.status_code == 400
-        assert "Content blocked by Model Armor" in str(exc_info.value.detail)
-
-def test_model_armor_should_block_content():
-    guardrail = ModelArmorGuardrail(
-        template_id="dummy-template",
-        project_id="dummy-project",
-        location="us-central1",
-        credentials=None,
-    )
-    # Block on inspectResult
-    armor_response_inspect = {
-        "sanitizationResult": {
-            "filterResults": [
-                {"sdpFilterResult": {"inspectResult": {"matchState": "MATCH_FOUND"}}}
-            ]
-        }
-    }
-    assert guardrail._should_block_content(armor_response_inspect)
-    # Block on deidentifyResult
-    armor_response_deidentify = {
-        "sanitizationResult": {
-            "filterResults": [
-                {"sdpFilterResult": {"deidentifyResult": {"matchState": "MATCH_FOUND"}}}
-            ]
-        }
-    }
-    assert guardrail._should_block_content(armor_response_deidentify)
-    # No block if neither
-    armor_response_none = {
-        "sanitizationResult": {
-            "filterResults": [
-                {"sdpFilterResult": {"inspectResult": {"matchState": "NO_MATCH_FOUND"}, "deidentifyResult": {"matchState": "NO_MATCH_FOUND"}}}
-            ]
-        }
-    }
-    assert not guardrail._should_block_content(armor_response_none)

tests/llm_translation/test_anthropic_completion.py

@@ -256,7 +256,6 @@ def test_anthropic_tool_streaming():
     for chunk in anthropic_chunk_list:
         parsed_chunk = response_iter.chunk_parser(chunk)
         if tool_use := parsed_chunk.get("tool_use"):
-
             # We only increment when a new block starts
             if tool_use.get("id") is not None:
                 correct_tool_index += 1
@@ -920,6 +919,14 @@ def test_anthropic_citations_api():
     citations = resp.choices[0].message.provider_specific_fields["citations"]
 
     assert citations is not None
+    if citations:
+        citation = citations[0][0]
+        assert "supported_text" in citation
+        assert "cited_text" in citation
+        assert "document_index" in citation
+        assert "document_title" in citation
+        assert "start_char_index" in citation
+        assert "end_char_index" in citation
 
 
 def test_anthropic_citations_api_streaming():
@@ -955,11 +962,9 @@ def test_anthropic_citations_api_streaming():
     has_citations = False
     for chunk in resp:
         print(f"returned chunk: {chunk}")
-        if (
-            chunk.choices[0].delta.provider_specific_fields
-            and "citation" in chunk.choices[0].delta.provider_specific_fields
-        ):
-            has_citations = True
+        if provider_specific_fields := chunk.choices[0].delta.provider_specific_fields:
+            if "citation" in provider_specific_fields:
+                has_citations = True
 
     assert has_citations
 

tests/local_testing/test_amazing_vertex_completion.py

@@ -2326,63 +2326,6 @@ def test_prompt_factory_nested():
         ), "'text' value not a string."
 
 
-def test_get_token_url():
-    from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
-        VertexLLM,
-    )
-
-    vertex_llm = VertexLLM()
-    vertex_ai_project = "pathrise-convert-1606954137718"
-    vertex_ai_location = "us-central1"
-    json_obj = get_vertex_ai_creds_json()
-    vertex_credentials = json.dumps(json_obj)
-
-    should_use_v1beta1_features = vertex_llm.is_using_v1beta1_features(
-        optional_params={"cached_content": "hi"}
-    )
-
-    assert should_use_v1beta1_features is True
-
-    _, url = vertex_llm._get_token_and_url(
-        auth_header=None,
-        vertex_project=vertex_ai_project,
-        vertex_location=vertex_ai_location,
-        vertex_credentials=vertex_credentials,
-        gemini_api_key="",
-        custom_llm_provider="vertex_ai_beta",
-        should_use_v1beta1_features=should_use_v1beta1_features,
-        api_base=None,
-        model="",
-        stream=False,
-    )
-
-    print("url=", url)
-
-    assert "/v1beta1/" in url
-
-    should_use_v1beta1_features = vertex_llm.is_using_v1beta1_features(
-        optional_params={"temperature": 0.1}
-    )
-
-    _, url = vertex_llm._get_token_and_url(
-        auth_header=None,
-        vertex_project=vertex_ai_project,
-        vertex_location=vertex_ai_location,
-        vertex_credentials=vertex_credentials,
-        gemini_api_key="",
-        custom_llm_provider="vertex_ai_beta",
-        should_use_v1beta1_features=should_use_v1beta1_features,
-        api_base=None,
-        model="",
-        stream=False,
-    )
-
-    print("url for normal request", url)
-
-    assert "v1beta1" not in url
-    assert "/v1/" in url
-
-    pass
 
 
 @pytest.mark.asyncio

tests/local_testing/test_completion.py

@@ -4321,20 +4321,6 @@ def test_langfuse_completion(monkeypatch):
     )
 
 
-def test_humanloop_completion(monkeypatch):
-    monkeypatch.setenv(
-        "HUMANLOOP_API_KEY", "hl_sk_59c1206e110c3f5b9985f0de4d23e7cbc79c4c4ae18c9f14"
-    )
-    litellm.set_verbose = True
-    resp = litellm.completion(
-        model="humanloop/gpt-3.5-turbo",
-        humanloop_api_key=os.getenv("HUMANLOOP_API_KEY"),
-        prompt_id="pr_nmSOVpEdyYPm2DrOwCoOm",
-        prompt_variables={"person": "John"},
-        messages=[{"role": "user", "content": "Tell me a joke."}],
-    )
-
-
 def test_completion_novita_ai():
     litellm.set_verbose = True
     messages = [

tests/test_litellm/integrations/test_langfuse.py

@@ -1,20 +1,222 @@
-import json
-import os
-import sys
-from typing import Optional
-
-# Adds the grandparent directory to sys.path to allow importing project modules
-sys.path.insert(0, os.path.abspath("../.."))
-
+import unittest
 import asyncio
-from unittest.mock import patch
-
+from unittest.mock import patch, MagicMock
+from typing import Optional
+import sys
+import os
+import datetime
+import json
 import pytest
-
 import litellm
 from litellm.integrations.langfuse import langfuse as langfuse_module
 from litellm.integrations.langfuse.langfuse import LangFuseLogger
 
+sys.path.insert(0, os.path.abspath("../.."))
+from litellm.integrations.langfuse.langfuse import LangFuseLogger
+# Import LangfuseUsageDetails directly from the module where it's defined
+from litellm.types.integrations.langfuse import *
+
+class TestLangfuseUsageDetails(unittest.TestCase):
+
+    def setUp(self):
+        # Set up environment variables for testing
+        self.env_patcher = patch.dict('os.environ', {
+            'LANGFUSE_SECRET_KEY': 'test-secret-key',
+            'LANGFUSE_PUBLIC_KEY': 'test-public-key',
+            'LANGFUSE_HOST': 'https://test.langfuse.com'
+        })
+        self.env_patcher.start()
+
+        # Create mock objects
+        self.mock_langfuse_client = MagicMock()
+        self.mock_langfuse_trace = MagicMock()
+        self.mock_langfuse_generation = MagicMock()
+
+        # Setup the trace and generation chain
+        self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation
+        self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace
+
+        # Mock the langfuse module that's imported locally in methods
+        self.langfuse_module_patcher = patch.dict('sys.modules', {'langfuse': MagicMock()})
+        self.mock_langfuse_module = self.langfuse_module_patcher.start()
+
+        # Create a mock for the langfuse module with version
+        self.mock_langfuse = MagicMock()
+        self.mock_langfuse.version = MagicMock()
+        self.mock_langfuse.version.__version__ = "3.0.0"  # Set a version that supports all features
+
+        # Mock the Langfuse class
+        self.mock_langfuse_class = MagicMock()
+        self.mock_langfuse_class.return_value = self.mock_langfuse_client
+
+        # Set up the sys.modules['langfuse'] mock
+        sys.modules['langfuse'] = self.mock_langfuse
+        sys.modules['langfuse'].Langfuse = self.mock_langfuse_class
+
+        # Mock the Langfuse client
+        self.mock_langfuse_client = MagicMock()
+        self.mock_langfuse_trace = MagicMock()
+        self.mock_langfuse_generation = MagicMock()
+
+        # Setup the trace and generation chain
+        self.mock_langfuse_trace.generation.return_value = self.mock_langfuse_generation
+        self.mock_langfuse_client.trace.return_value = self.mock_langfuse_trace
+
+        # Mock the Langfuse class
+        self.mock_langfuse_class = MagicMock()
+        self.mock_langfuse_class.return_value = self.mock_langfuse_client
+        self.mock_langfuse.Langfuse = self.mock_langfuse_class
+
+        # Create the logger
+        self.logger = LangFuseLogger()
+
+        # Add the log_event_on_langfuse method to the instance
+        def log_event_on_langfuse(self, kwargs, response_obj, start_time=None, end_time=None, user_id=None, level="DEFAULT", status_message=None):
+            # This implementation calls _log_langfuse_v2 directly
+            return self._log_langfuse_v2(
+                user_id=user_id,
+                metadata=kwargs.get("litellm_params", {}).get("metadata", {}),
+                litellm_params=kwargs.get("litellm_params", {}),
+                output=None,
+                start_time=start_time,
+                end_time=end_time,
+                kwargs=kwargs,
+                optional_params=kwargs.get("optional_params", {}),
+                input=None,
+                response_obj=response_obj,
+                level=level,
+                litellm_call_id=kwargs.get("litellm_call_id", None),
+                print_verbose=True  # Add the missing parameter
+            )
+
+        # Bind the method to the instance
+        import types
+        self.logger.log_event_on_langfuse = types.MethodType(log_event_on_langfuse, self.logger)
+
+        # Make sure _is_langfuse_v2 returns True
+        def mock_is_langfuse_v2(self):
+            return True
+
+        self.logger._is_langfuse_v2 = types.MethodType(mock_is_langfuse_v2, self.logger)
+
+    def tearDown(self):
+        self.env_patcher.stop()
+        self.langfuse_module_patcher.stop()
+
+    def test_langfuse_usage_details_type(self):
+        """Test that LangfuseUsageDetails TypedDict is properly defined with the correct fields"""
+        # Create an instance of LangfuseUsageDetails
+        usage_details: LangfuseUsageDetails = {
+            "input": 10,
+            "output": 20,
+            "cache_creation_input_tokens": 5,
+            "cache_read_input_tokens": 3
+        }
+
+        # Verify all fields are present
+        self.assertEqual(usage_details["input"], 10)
+        self.assertEqual(usage_details["output"], 20)
+        self.assertEqual(usage_details["cache_creation_input_tokens"], 5)
+        self.assertEqual(usage_details["cache_read_input_tokens"], 3)
+
+        # Test with all fields (all fields are required in TypedDict by default)
+        minimal_usage_details: LangfuseUsageDetails = {
+            "input": 10,
+            "output": 20,
+            "cache_creation_input_tokens": 0,
+            "cache_read_input_tokens": 0
+        }
+
+        self.assertEqual(minimal_usage_details["input"], 10)
+        self.assertEqual(minimal_usage_details["output"], 20)
+
+    def test_log_langfuse_v2_usage_details(self):
+        """Test that usage_details in _log_langfuse_v2 is correctly typed and assigned"""
+        # Create a mock response object with usage information
+        response_obj = MagicMock()
+        response_obj.usage = MagicMock()
+        response_obj.usage.prompt_tokens = 15
+        response_obj.usage.completion_tokens = 25
+
+        # Add the cache token attributes using get method
+        def mock_get(key, default=None):
+            if key == 'cache_creation_input_tokens':
+                return 7
+            elif key == 'cache_read_input_tokens':
+                return 4
+            return default
+
+        response_obj.usage.get = mock_get
+
+        # Create kwargs for the log_event method
+        kwargs = {
+            "model": "gpt-4",
+            "messages": [{"role": "user", "content": "Hello"}],
+            "litellm_params": {"metadata": {}}
+        }
+
+        # Create start and end times
+        start_time = datetime.datetime.now()
+        end_time = start_time + datetime.timedelta(seconds=1)
+
+        # Call the log_event method
+        with patch.object(self.logger, '_log_langfuse_v2') as mock_log_langfuse_v2:
+            self.logger.log_event_on_langfuse(
+                kwargs=kwargs,
+                response_obj=response_obj,
+                start_time=start_time,
+                end_time=end_time
+            )
+
+            # Check if _log_langfuse_v2 was called
+            mock_log_langfuse_v2.assert_called_once()
+
+            # Get the arguments passed to _log_langfuse_v2
+            call_args = mock_log_langfuse_v2.call_args[1]
+
+            # Verify response_obj was passed correctly
+            self.assertEqual(call_args["response_obj"], response_obj)
+
+    def test_langfuse_usage_details_optional_fields(self):
+        """Test that LangfuseUsageDetails fields are properly defined as Optional"""
+        # Create an instance with None values for optional fields
+        usage_details: LangfuseUsageDetails = {
+            "input": 10,
+            "output": 20,
+            "cache_creation_input_tokens": None,
+            "cache_read_input_tokens": None
+        }
+
+        # Verify fields can be None
+        self.assertEqual(usage_details["input"], 10)
+        self.assertEqual(usage_details["output"], 20)
+        self.assertIsNone(usage_details["cache_creation_input_tokens"])
+        self.assertIsNone(usage_details["cache_read_input_tokens"])
+
+    def test_langfuse_usage_details_structure(self):
+        """Test that LangfuseUsageDetails has the correct structure as defined in the commit"""
+        # This test directly verifies the structure of the TypedDict
+        # without relying on the LangFuseLogger class
+
+        # Create a dictionary that matches the LangfuseUsageDetails structure
+        usage_details = {
+            "input": 15,
+            "output": 25,
+            "cache_creation_input_tokens": 7,
+            "cache_read_input_tokens": 4
+        }
+
+        # Verify the structure matches what we expect
+        self.assertIn("input", usage_details)
+        self.assertIn("output", usage_details)
+        self.assertIn("cache_creation_input_tokens", usage_details)
+        self.assertIn("cache_read_input_tokens", usage_details)
+
+        # Verify the values
+        self.assertEqual(usage_details["input"], 15)
+        self.assertEqual(usage_details["output"], 25)
+        self.assertEqual(usage_details["cache_creation_input_tokens"], 7)
+        self.assertEqual(usage_details["cache_read_input_tokens"], 4)
 
 def test_max_langfuse_clients_limit():
     """

tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py

@@ -183,7 +183,30 @@ def test_extract_response_content_with_citations():
     }
 
     _, citations, _, _, _ = config.extract_response_content(completion_response)
-    assert citations is not None
+    assert citations == [
+        [
+            {
+                "type": "char_location",
+                "cited_text": "The grass is green. ",
+                "document_index": 0,
+                "document_title": "My Document",
+                "start_char_index": 0,
+                "end_char_index": 20,
+                "supported_text": "the grass is green",
+            },
+        ],
+        [
+            {
+                "type": "char_location",
+                "cited_text": "The sky is blue.",
+                "document_index": 0,
+                "document_title": "My Document",
+                "start_char_index": 20,
+                "end_char_index": 36,
+                "supported_text": "the sky is blue",
+            },
+        ],
+    ]
 
 
 def test_map_tool_helper():

tests/test_litellm/llms/custom_httpx/test_http_handler.py

@@ -29,16 +29,17 @@ async def test_ssl_security_level(monkeypatch):
 
         # Get the transport (should be LiteLLMAiohttpTransport)
         transport = client.client._transport
+        assert isinstance(transport, LiteLLMAiohttpTransport)
 
         # Get the aiohttp ClientSession
         client_session = transport._get_valid_client_session()
 
         # Get the connector from the session
         connector = client_session.connector
+        assert isinstance(connector, TCPConnector)
 
         # Get the SSL context from the connector
         ssl_context = connector._ssl
-        print("ssl_context", ssl_context)
 
         # Verify that the SSL context exists and has the correct cipher string
         assert isinstance(ssl_context, ssl.SSLContext)
@@ -108,20 +109,19 @@ async def test_ssl_verification_with_aiohttp_transport():
     # Create a test SSL context
     litellm_async_client = AsyncHTTPHandler(ssl_verify=False)
 
-    transport_connector = (
-        litellm_async_client.client._transport._get_valid_client_session().connector
-    )
-    print("transport_connector", transport_connector)
-    print("transport_connector._ssl", transport_connector._ssl)
+    transport = litellm_async_client.client._transport
+    assert isinstance(transport, LiteLLMAiohttpTransport)
+    transport_connector = transport._get_valid_client_session().connector
+    assert isinstance(transport_connector, TCPConnector)
 
     aiohttp_session = aiohttp.ClientSession(
         connector=aiohttp.TCPConnector(verify_ssl=False)
     )
-    print("aiohttp_session", aiohttp_session)
-    print("aiohttp_session._ssl", aiohttp_session.connector._ssl)
+    aiohttp_connector = aiohttp_session.connector
+    assert isinstance(aiohttp_connector, aiohttp.TCPConnector)
 
     # assert both litellm transport and aiohttp session have ssl_verify=False
-    assert transport_connector._ssl == aiohttp_session.connector._ssl
+    assert transport_connector._ssl == aiohttp_connector._ssl
 
 
 @pytest.mark.asyncio
@@ -183,3 +183,219 @@ def test_get_ssl_configuration_integration():
     # Verify it has basic SSL context properties
     assert ssl_context.protocol is not None
     assert ssl_context.verify_mode is not None
+
+
+# Session Reuse Tests
+class MockClientSession:
+    """Mock ClientSession that is not callable"""
+    def __init__(self):
+        self.closed = False
+
+@pytest.mark.asyncio
+async def test_create_aiohttp_transport_with_shared_session():
+    """Test that _create_aiohttp_transport reuses shared session when provided"""
+    from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+    
+    # Create a mock shared session that's not callable
+    mock_session = MockClientSession()
+    
+    # Test with shared session
+    transport = AsyncHTTPHandler._create_aiohttp_transport(
+        shared_session=mock_session  # type: ignore
+    )
+    
+    # Verify the transport uses the shared session directly
+    assert transport.client is mock_session
+    assert not callable(transport.client)  # Should not be callable
+
+
+@pytest.mark.asyncio
+async def test_create_aiohttp_transport_without_shared_session():
+    """Test that _create_aiohttp_transport creates new session when none provided"""
+    from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+    
+    # Test without shared session
+    transport = AsyncHTTPHandler._create_aiohttp_transport(shared_session=None)
+    
+    # Verify the transport uses a lambda function (for backward compatibility)
+    assert callable(transport.client)  # Should be a lambda function
+
+
+@pytest.mark.asyncio
+async def test_create_aiohttp_transport_with_closed_session():
+    """Test that _create_aiohttp_transport creates new session when shared session is closed"""
+    from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+    
+    # Create a mock closed session
+    mock_session = MockClientSession()
+    mock_session.closed = True
+    
+    # Test with closed session
+    transport = AsyncHTTPHandler._create_aiohttp_transport(
+        shared_session=mock_session  # type: ignore
+    )
+    
+    # Verify the transport creates a new session (lambda function)
+    assert callable(transport.client)  # Should be a lambda function
+
+
+@pytest.mark.asyncio
+async def test_async_handler_with_shared_session():
+    """Test AsyncHTTPHandler initialization with shared session"""
+    from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+    
+    # Create a mock shared session
+    mock_session = MockClientSession()
+    
+    # Create handler with shared session
+    handler = AsyncHTTPHandler(shared_session=mock_session)  # type: ignore
+    
+    # Verify the handler was created successfully
+    assert handler is not None
+    assert handler.client is not None
+
+
+@pytest.mark.asyncio
+async def test_get_async_httpx_client_with_shared_session():
+    """Test get_async_httpx_client with shared session"""
+    from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+    from litellm.types.utils import LlmProviders
+    
+    # Create a mock shared session
+    mock_session = MockClientSession()
+    
+    # Test with shared session
+    client = get_async_httpx_client(
+        llm_provider=LlmProviders.ANTHROPIC,
+        shared_session=mock_session  # type: ignore
+    )
+    
+    # Verify the client was created successfully
+    assert client is not None
+    assert isinstance(client, AsyncHTTPHandler)
+
+
+@pytest.mark.asyncio
+async def test_get_async_httpx_client_without_shared_session():
+    """Test get_async_httpx_client without shared session (backward compatibility)"""
+    from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+    from litellm.types.utils import LlmProviders
+    
+    # Test without shared session
+    client = get_async_httpx_client(
+        llm_provider=LlmProviders.ANTHROPIC,
+        shared_session=None
+    )
+    
+    # Verify the client was created successfully
+    assert client is not None
+    assert isinstance(client, AsyncHTTPHandler)
+
+
+@pytest.mark.asyncio
+async def test_session_reuse_chain():
+    """Test that session is properly passed through the entire call chain"""
+    from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+    
+    # Create a mock shared session
+    mock_session = MockClientSession()
+    
+    # Test the entire chain
+    transport = AsyncHTTPHandler._create_async_transport(
+        shared_session=mock_session  # type: ignore
+    )
+    
+    # Verify the transport was created
+    assert transport is not None
+    
+    # Test AsyncHTTPHandler creation
+    handler = AsyncHTTPHandler(shared_session=mock_session)  # type: ignore
+    assert handler is not None
+
+
+def test_shared_session_parameter_in_acompletion():
+    """Test that acompletion function accepts shared_session parameter"""
+    import inspect
+    from litellm.main import acompletion
+    
+    # Get the function signature
+    sig = inspect.signature(acompletion)
+    params = list(sig.parameters.keys())
+    
+    # Verify shared_session parameter exists
+    assert 'shared_session' in params
+    
+    # Verify the parameter type annotation
+    shared_session_param = sig.parameters['shared_session']
+    assert 'ClientSession' in str(shared_session_param.annotation)
+
+
+def test_shared_session_parameter_in_completion():
+    """Test that completion function accepts shared_session parameter"""
+    import inspect
+    from litellm.main import completion
+    
+    # Get the function signature
+    sig = inspect.signature(completion)
+    params = list(sig.parameters.keys())
+    
+    # Verify shared_session parameter exists
+    assert 'shared_session' in params
+    
+    # Verify the parameter type annotation
+    shared_session_param = sig.parameters['shared_session']
+    assert 'ClientSession' in str(shared_session_param.annotation)
+
+
+@pytest.mark.asyncio
+async def test_session_reuse_integration():
+    """Integration test for session reuse functionality"""
+    from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+    from litellm.types.utils import LlmProviders
+    
+    # Create a mock session
+    mock_session = MockClientSession()
+    
+    # Create two clients with the same session
+    client1 = get_async_httpx_client(
+        llm_provider=LlmProviders.ANTHROPIC,
+        shared_session=mock_session  # type: ignore
+    )
+    
+    client2 = get_async_httpx_client(
+        llm_provider=LlmProviders.OPENAI,
+        shared_session=mock_session  # type: ignore
+    )
+    
+    # Both clients should be created successfully
+    assert client1 is not None
+    assert client2 is not None
+    
+    # Both should be AsyncHTTPHandler instances
+    assert isinstance(client1, AsyncHTTPHandler)
+    assert isinstance(client2, AsyncHTTPHandler)
+    
+    # Clean up
+    await client1.close()
+    await client2.close()
+
+
+@pytest.mark.asyncio
+async def test_session_validation():
+    """Test that session validation works correctly"""
+    from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+    
+    # Test with None session
+    transport1 = AsyncHTTPHandler._create_aiohttp_transport(shared_session=None)
+    assert callable(transport1.client)  # Should create lambda
+    
+    # Test with closed session
+    mock_closed_session = MockClientSession()
+    mock_closed_session.closed = True
+    transport2 = AsyncHTTPHandler._create_aiohttp_transport(shared_session=mock_closed_session)  # type: ignore
+    assert callable(transport2.client)  # Should create lambda
+    
+    # Test with valid session
+    mock_valid_session = MockClientSession()
+    transport3 = AsyncHTTPHandler._create_aiohttp_transport(shared_session=mock_valid_session)  # type: ignore
+    assert transport3.client is mock_valid_session  # Should reuse session

tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py

@@ -801,3 +801,59 @@ def test_fix_enum_empty_strings():
     # 3. Other properties preserved
     assert input_schema["properties"]["user_agent_type"]["type"] == "string"
     assert input_schema["properties"]["user_agent_type"]["description"] == "Device type for user agent"
+
+
+def test_get_token_url():
+    from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
+        VertexLLM,
+    )
+
+    vertex_llm = VertexLLM()
+    vertex_ai_project = "pathrise-convert-1606954137718"
+    vertex_ai_location = "us-central1"
+    vertex_credentials = ""
+
+    should_use_v1beta1_features = vertex_llm.is_using_v1beta1_features(
+        optional_params={"cached_content": "hi"}
+    )
+
+    _, url = vertex_llm._get_token_and_url(
+        auth_header=None,
+        vertex_project=vertex_ai_project,
+        vertex_location=vertex_ai_location,
+        vertex_credentials=vertex_credentials,
+        gemini_api_key="",
+        custom_llm_provider="vertex_ai_beta",
+        should_use_v1beta1_features=should_use_v1beta1_features,
+        api_base=None,
+        model="",
+        stream=False,
+    )
+
+    print("url=", url)
+
+
+
+    should_use_v1beta1_features = vertex_llm.is_using_v1beta1_features(
+        optional_params={"temperature": 0.1}
+    )
+
+    _, url = vertex_llm._get_token_and_url(
+        auth_header=None,
+        vertex_project=vertex_ai_project,
+        vertex_location=vertex_ai_location,
+        vertex_credentials=vertex_credentials,
+        gemini_api_key="",
+        custom_llm_provider="vertex_ai_beta",
+        should_use_v1beta1_features=should_use_v1beta1_features,
+        api_base=None,
+        model="",
+        stream=False,
+    )
+
+    print("url for normal request", url)
+
+    assert "v1beta1" not in url
+    assert "/v1/" in url
+
+    pass

tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py

@@ -102,7 +102,7 @@ async def test_get_tools_from_mcp_servers_continues_when_one_server_fails():
         working_server if server_id == "working_server" else failing_server
     )
 
-    async def mock_get_tools_from_server(server, mcp_auth_header=None, add_prefix=True):
+    async def mock_get_tools_from_server(server, mcp_auth_header=None):
         if server.name == "working_server":
             # Working server returns tools
             tool1 = MagicMock()
@@ -184,7 +184,7 @@ async def test_get_tools_from_mcp_servers_handles_all_servers_failing():
         failing_server1 if server_id == "failing_server1" else failing_server2
     )
 
-    async def mock_get_tools_from_server(server, mcp_auth_header=None, add_prefix=True):
+    async def mock_get_tools_from_server(server, mcp_auth_header=None):
         # All servers fail
         raise Exception(f"Server {server.name} connection failed")
 
@@ -448,121 +448,3 @@ async def test_mcp_routing_with_conflicting_alias_and_group_name():
     assert (
         called_servers[0].server_id == specific_server.server_id
     ), "Should have contacted the specific server alias, not the group."
-
-
-@pytest.mark.asyncio
-async def test_list_tools_single_server_unprefixed_names():
-    """When only one MCP server is allowed, list tools should return unprefixed names."""
-    try:
-        from litellm.proxy._experimental.mcp_server.server import (
-            _get_tools_from_mcp_servers,
-            set_auth_context,
-        )
-    except ImportError:
-        pytest.skip("MCP server not available")
-
-    # Mock user auth
-    user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user")
-    set_auth_context(user_api_key_auth)
-
-    # One allowed server
-    server = MagicMock()
-    server.server_id = "server1"
-    server.name = "Zapier MCP"
-    server.alias = "zapier"
-
-    # Mock manager: allow just one server and return a tool based on add_prefix flag
-    mock_manager = MagicMock()
-    mock_manager.get_allowed_mcp_servers = AsyncMock(return_value=["server1"])
-    mock_manager.get_mcp_server_by_id = (
-        lambda server_id: server if server_id == "server1" else None
-    )
-
-    async def mock_get_tools_from_server(
-        server, mcp_auth_header=None, add_prefix=False
-    ):
-        tool = MagicMock()
-        tool.name = f"{server.alias}-toolA" if add_prefix else "toolA"
-        tool.description = "desc"
-        tool.inputSchema = {}
-        return [tool]
-
-    mock_manager._get_tools_from_server = mock_get_tools_from_server
-
-    with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
-        mock_manager,
-    ):
-        tools = await _get_tools_from_mcp_servers(
-            user_api_key_auth=user_api_key_auth,
-            mcp_auth_header=None,
-            mcp_servers=None,
-            mcp_server_auth_headers=None,
-        )
-
-    # Should be unprefixed since only one server is allowed
-    assert len(tools) == 1
-    assert tools[0].name == "toolA"
-
-
-@pytest.mark.asyncio
-async def test_list_tools_multiple_servers_prefixed_names():
-    """When multiple MCP servers are allowed, list tools should return prefixed names."""
-    try:
-        from litellm.proxy._experimental.mcp_server.server import (
-            _get_tools_from_mcp_servers,
-            set_auth_context,
-        )
-    except ImportError:
-        pytest.skip("MCP server not available")
-
-    # Mock user auth
-    user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user")
-    set_auth_context(user_api_key_auth)
-
-    # Two allowed servers
-    server1 = MagicMock()
-    server1.server_id = "server1"
-    server1.name = "Zapier MCP"
-    server1.alias = "zapier"
-
-    server2 = MagicMock()
-    server2.server_id = "server2"
-    server2.name = "Jira MCP"
-    server2.alias = "jira"
-
-    # Mock manager
-    mock_manager = MagicMock()
-    mock_manager.get_allowed_mcp_servers = AsyncMock(
-        return_value=["server1", "server2"]
-    )
-    mock_manager.get_mcp_server_by_id = (
-        lambda server_id: server1 if server_id == "server1" else server2
-    )
-
-    async def mock_get_tools_from_server(
-        server, mcp_auth_header=None, add_prefix=True
-    ):
-        tool = MagicMock()
-        # When multiple servers, add_prefix should be True -> prefixed names
-        tool.name = f"{server.alias}-toolA" if add_prefix else "toolA"
-        tool.description = "desc"
-        tool.inputSchema = {}
-        return [tool]
-
-    mock_manager._get_tools_from_server = mock_get_tools_from_server
-
-    with patch(
-        "litellm.proxy._experimental.mcp_server.server.global_mcp_server_manager",
-        mock_manager,
-    ):
-        tools = await _get_tools_from_mcp_servers(
-            user_api_key_auth=user_api_key_auth,
-            mcp_auth_header=None,
-            mcp_servers=None,
-            mcp_server_auth_headers=None,
-        )
-
-    # Should be prefixed since multiple servers are allowed
-    names = sorted([t.name for t in tools])
-    assert names == ["jira-toolA", "zapier-toolA"]

tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py

@@ -420,112 +420,6 @@ class TestMCPServerManager:
         assert result["status"] == "healthy"
         assert result["tools_count"] == 1
 
-    @pytest.mark.asyncio
-    async def test_get_tools_from_server_add_prefix(self):
-        """Verify _get_tools_from_server respects add_prefix True/False."""
-        manager = MCPServerManager()
-
-        # Create a minimal server with alias used as prefix
-        server = MCPServer(
-            server_id="zapier",
-            name="zapier",
-            transport=MCPTransport.http,
-        )
-
-        # Mock client creation and fetching tools
-        manager._create_mcp_client = MagicMock(return_value=object())
-
-        # Tools returned upstream (unprefixed from provider)
-        upstream_tool = MagicMock()
-        upstream_tool.name = "send_email"
-        upstream_tool.description = "Send an email"
-        upstream_tool.inputSchema = {}
-
-        manager._fetch_tools_with_timeout = AsyncMock(return_value=[upstream_tool])
-
-        # Case 1: add_prefix=True (default for multi-server) -> expect prefixed
-        tools_prefixed = await manager._get_tools_from_server(server, add_prefix=True)
-        assert len(tools_prefixed) == 1
-        assert tools_prefixed[0].name == "zapier-send_email"
-
-        # Case 2: add_prefix=False (single-server) -> expect unprefixed
-        tools_unprefixed = await manager._get_tools_from_server(
-            server, add_prefix=False
-        )
-        assert len(tools_unprefixed) == 1
-        assert tools_unprefixed[0].name == "send_email"
-
-    def test_create_prefixed_tools_updates_mapping_for_both_forms(self):
-        """_create_prefixed_tools should populate mapping for prefixed and original names even when not adding prefix in output."""
-        manager = MCPServerManager()
-
-        server = MCPServer(
-            server_id="jira",
-            name="jira",
-            transport=MCPTransport.http,
-        )
-
-        # Input tools as would come from upstream
-        t1 = MagicMock()
-        t1.name = "create_issue"
-        t1.description = ""
-        t1.inputSchema = {}
-        t2 = MagicMock()
-        t2.name = "close_issue"
-        t2.description = ""
-        t2.inputSchema = {}
-
-        # Do not add prefix in returned objects
-        out_tools = manager._create_prefixed_tools([t1, t2], server, add_prefix=False)
-
-        # Returned names should be unprefixed
-        names = sorted([t.name for t in out_tools])
-        assert names == ["close_issue", "create_issue"]
-
-        # Mapping should include both original and prefixed names -> resolves calls either way
-        assert manager.tool_name_to_mcp_server_name_mapping["create_issue"] == "jira"
-        assert (
-            manager.tool_name_to_mcp_server_name_mapping["jira-create_issue"] == "jira"
-        )
-        assert manager.tool_name_to_mcp_server_name_mapping["close_issue"] == "jira"
-        assert (
-            manager.tool_name_to_mcp_server_name_mapping["jira-close_issue"] == "jira"
-        )
-
-    def test_get_mcp_server_from_tool_name_with_prefixed_and_unprefixed(self):
-        """After mapping is populated, manager resolves both prefixed and unprefixed tool names to the same server."""
-        manager = MCPServerManager()
-
-        server = MCPServer(
-            server_id="zapier",
-            name="zapier",
-            server_name="zapier",
-            transport=MCPTransport.http,
-        )
-
-        # Register server so resolution can find it
-        manager.registry = {server.server_id: server}
-
-        # Populate mapping (add_prefix value doesn't matter for mapping population)
-        base_tool = MagicMock()
-        base_tool.name = "create_zap"
-        base_tool.description = ""
-        base_tool.inputSchema = {}
-        _ = manager._create_prefixed_tools([base_tool], server, add_prefix=False)
-
-        # Unprefixed resolution
-        resolved_server_unpref = manager._get_mcp_server_from_tool_name("create_zap")
-        print(resolved_server_unpref)
-        assert resolved_server_unpref is not None
-        assert resolved_server_unpref.server_id == server.server_id
-
-        # Prefixed resolution
-        resolved_server_pref = manager._get_mcp_server_from_tool_name(
-            "zapier-create_zap"
-        )
-        assert resolved_server_pref is not None
-        assert resolved_server_pref.server_id == server.server_id
-
 
 if __name__ == "__main__":
     pytest.main([__file__])

tests/test_litellm/proxy/client/cli/test_auth_commands.py

@@ -159,13 +159,13 @@ class TestTokenUtilities:
             'user_id': 'test-user'
         }
         
-        with patch('litellm.proxy.client.cli.commands.auth.load_token', return_value=token_data):
+        with patch('litellm.litellm_core_utils.cli_token_utils.load_cli_token', return_value=token_data):
             result = get_stored_api_key()
             assert result == 'test-api-key-123'
 
     def test_get_stored_api_key_no_token(self):
         """Test getting stored API key when no token exists"""
-        with patch('litellm.proxy.client.cli.commands.auth.load_token', return_value=None):
+        with patch('litellm.litellm_core_utils.cli_token_utils.load_cli_token', return_value=None):
             result = get_stored_api_key()
             assert result is None
 
@@ -175,7 +175,7 @@ class TestTokenUtilities:
             'user_id': 'test-user'
         }
         
-        with patch('litellm.proxy.client.cli.commands.auth.load_token', return_value=token_data):
+        with patch('litellm.litellm_core_utils.cli_token_utils.load_cli_token', return_value=token_data):
             result = get_stored_api_key()
             assert result is None
 
@@ -471,7 +471,7 @@ class TestCLIKeyRegenerationFlow:
             
             assert result.exit_code == 0
             assert "✅ Login successful!" in result.output
-            assert "API Key: sk-regenerated-key-456" in result.output
+            assert "API Key: sk-regenerated-key-4..." in result.output
             
             # Verify existing key was retrieved
             mock_get_stored.assert_called_once()
@@ -486,7 +486,9 @@ class TestCLIKeyRegenerationFlow:
             
             # Verify polling was done with correct session key
             mock_get.assert_called()
-            poll_url = mock_get.call_args[0][0]
+            # Check that the polling URL was called (should be the first call)
+            first_call_args = mock_get.call_args_list[0]
+            poll_url = first_call_args[0][0]
             assert "sk-new-session-uuid-789" in poll_url
             
             # Verify regenerated key was saved

tests/test_litellm/proxy/client/cli/test_chat_commands.py

@@ -1,248 +0,0 @@
-import json
-import os
-import sys
-from unittest.mock import MagicMock, patch
-
-import pytest
-import requests
-from click.testing import CliRunner
-
-sys.path.insert(
-    0, os.path.abspath("../../..")
-)  # Adds the parent directory to the system path
-
-
-from litellm.proxy.client.cli.main import cli
-
-
-@pytest.fixture
-def mock_chat_client():
-    with patch("litellm.proxy.client.cli.commands.chat.ChatClient") as mock:
-        yield mock
-
-
-@pytest.fixture
-def cli_runner():
-    return CliRunner()
-
-
-def test_chat_completions_success(cli_runner, mock_chat_client):
-    # Mock response data
-    mock_response = {
-        "id": "chatcmpl-123",
-        "object": "chat.completion",
-        "created": 1677858242,
-        "model": "gpt-4",
-        "choices": [
-            {
-                "message": {
-                    "role": "assistant",
-                    "content": "Hello! How can I help you today?",
-                },
-                "finish_reason": "stop",
-                "index": 0,
-            }
-        ],
-    }
-    mock_instance = mock_chat_client.return_value
-    mock_instance.completions.return_value = mock_response
-
-    # Run command
-    result = cli_runner.invoke(
-        cli,
-        [
-            "chat",
-            "completions",
-            "gpt-4",
-            "-m",
-            "user:Hello!",
-            "--temperature",
-            "0.7",
-            "--max-tokens",
-            "100",
-        ],
-    )
-
-    # Verify
-    assert result.exit_code == 0
-    output_data = json.loads(result.output)
-    assert output_data == mock_response
-    mock_instance.completions.assert_called_once_with(
-        model="gpt-4",
-        messages=[{"role": "user", "content": "Hello!"}],
-        temperature=0.7,
-        max_tokens=100,
-        top_p=None,
-        n=None,
-        presence_penalty=None,
-        frequency_penalty=None,
-        user=None,
-    )
-
-
-def test_chat_completions_multiple_messages(cli_runner, mock_chat_client):
-    # Mock response data
-    mock_response = {
-        "id": "chatcmpl-123",
-        "object": "chat.completion",
-        "created": 1677858242,
-        "model": "gpt-4",
-        "choices": [
-            {
-                "message": {
-                    "role": "assistant",
-                    "content": "Paris has a population of about 2.2 million.",
-                },
-                "finish_reason": "stop",
-                "index": 0,
-            }
-        ],
-    }
-    mock_instance = mock_chat_client.return_value
-    mock_instance.completions.return_value = mock_response
-
-    # Run command
-    result = cli_runner.invoke(
-        cli,
-        [
-            "chat",
-            "completions",
-            "gpt-4",
-            "-m",
-            "system:You are a helpful assistant",
-            "-m",
-            "user:What's the population of Paris?",
-        ],
-    )
-
-    # Verify
-    assert result.exit_code == 0
-    output_data = json.loads(result.output)
-    assert output_data == mock_response
-    mock_instance.completions.assert_called_once_with(
-        model="gpt-4",
-        messages=[
-            {"role": "system", "content": "You are a helpful assistant"},
-            {"role": "user", "content": "What's the population of Paris?"},
-        ],
-        temperature=None,
-        max_tokens=None,
-        top_p=None,
-        n=None,
-        presence_penalty=None,
-        frequency_penalty=None,
-        user=None,
-    )
-
-
-def test_chat_completions_no_messages(cli_runner, mock_chat_client):
-    # Run command without any messages
-    result = cli_runner.invoke(cli, ["chat", "completions", "gpt-4"])
-
-    # Verify
-    assert result.exit_code == 2
-    assert "At least one message is required" in result.output
-    mock_instance = mock_chat_client.return_value
-    mock_instance.completions.assert_not_called()
-
-
-def test_chat_completions_invalid_message_format(cli_runner, mock_chat_client):
-    # Run command with invalid message format
-    result = cli_runner.invoke(
-        cli, ["chat", "completions", "gpt-4", "-m", "invalid-format"]
-    )
-
-    # Verify
-    assert result.exit_code == 2
-    assert "Invalid message format" in result.output
-    mock_instance = mock_chat_client.return_value
-    mock_instance.completions.assert_not_called()
-
-
-def test_chat_completions_http_error(cli_runner, mock_chat_client):
-    # Mock HTTP error
-    mock_instance = mock_chat_client.return_value
-    mock_error_response = MagicMock()
-    mock_error_response.status_code = 400
-    mock_error_response.json.return_value = {
-        "error": "Invalid request",
-        "message": "Invalid model specified",
-    }
-    mock_instance.completions.side_effect = requests.exceptions.HTTPError(
-        response=mock_error_response
-    )
-
-    # Run command
-    result = cli_runner.invoke(
-        cli, ["chat", "completions", "invalid-model", "-m", "user:Hello"]
-    )
-
-    # Verify
-    assert result.exit_code == 1
-    assert "Error: HTTP 400" in result.output
-    assert "Invalid request" in result.output
-    assert "Invalid model specified" in result.output
-
-
-def test_chat_completions_all_parameters(cli_runner, mock_chat_client):
-    # Mock response data
-    mock_response = {
-        "id": "chatcmpl-123",
-        "object": "chat.completion",
-        "created": 1677858242,
-        "model": "gpt-4",
-        "choices": [
-            {
-                "message": {
-                    "role": "assistant",
-                    "content": "Response with all parameters set",
-                },
-                "finish_reason": "stop",
-                "index": 0,
-            }
-        ],
-    }
-    mock_instance = mock_chat_client.return_value
-    mock_instance.completions.return_value = mock_response
-
-    # Run command with all available parameters
-    result = cli_runner.invoke(
-        cli,
-        [
-            "chat",
-            "completions",
-            "gpt-4",
-            "-m",
-            "user:Test message",
-            "--temperature",
-            "0.7",
-            "--top-p",
-            "0.9",
-            "--n",
-            "1",
-            "--max-tokens",
-            "100",
-            "--presence-penalty",
-            "0.5",
-            "--frequency-penalty",
-            "0.5",
-            "--user",
-            "test-user",
-        ],
-    )
-
-    # Verify
-    assert result.exit_code == 0
-    output_data = json.loads(result.output)
-    assert output_data == mock_response
-    mock_instance.completions.assert_called_once_with(
-        model="gpt-4",
-        messages=[{"role": "user", "content": "Test message"}],
-        temperature=0.7,
-        top_p=0.9,
-        n=1,
-        max_tokens=100,
-        presence_penalty=0.5,
-        frequency_penalty=0.5,
-        user="test-user",
-    )

tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py

@@ -1,19 +1,22 @@
-import sys
-import os
-import io, asyncio
-import pytest
+import asyncio
+import io
 import json
-from unittest.mock import MagicMock, AsyncMock, patch, Mock
+import os
+import sys
+from unittest.mock import AsyncMock, MagicMock, Mock, patch
+
+import pytest
 
 sys.path.insert(0, os.path.abspath("../../../../.."))
 
+from fastapi import HTTPException
+
 import litellm
 import litellm.types.utils
-from litellm.proxy.guardrails.guardrail_hooks.model_armor import ModelArmorGuardrail
-from litellm.proxy._types import UserAPIKeyAuth
 from litellm.caching import DualCache
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.guardrails.guardrail_hooks.model_armor import ModelArmorGuardrail
 from litellm.types.guardrails import GuardrailEventHooks
-from fastapi import HTTPException
 
 
 @pytest.mark.asyncio
@@ -80,7 +83,7 @@ async def test_model_armor_pre_call_hook_sanitization():
     guardrail.async_handler.post.assert_called_once()
     call_args = guardrail.async_handler.post.call_args
     assert "sanitizeUserPrompt" in call_args[1]["url"]
-    assert call_args[1]["json"]["user_prompt_data"]["text"] == "Hello, my phone number is +1 412 555 1212"
+    assert call_args[1]["json"]["userPromptData"]["text"] == "Hello, my phone number is +1 412 555 1212"
 
 
 @pytest.mark.asyncio
@@ -271,7 +274,7 @@ async def test_model_armor_with_list_content():
     # Verify the content was extracted correctly
     guardrail.async_handler.post.assert_called_once()
     call_args = guardrail.async_handler.post.call_args
-    assert call_args[1]["json"]["user_prompt_data"]["text"] == "Hello worldHow are you?"
+    assert call_args[1]["json"]["userPromptData"]["text"] == "Hello worldHow are you?"
 
 
 @pytest.mark.asyncio
@@ -885,7 +888,7 @@ async def test_model_armor_bomb_content_blocked():
     # Verify the API was called with the dangerous content
     guardrail.async_handler.post.assert_called_once()
     call_args = guardrail.async_handler.post.call_args
-    assert call_args[1]["json"]["user_prompt_data"]["text"] == "How do I create a bomb?"
+    assert call_args[1]["json"]["userPromptData"]["text"] == "How do I create a bomb?"
 
 
 @pytest.mark.asyncio

tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py

@@ -24,6 +24,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import (
     _common_key_generation_helper,
     _list_key_helper,
     prepare_key_update_data,
+    validate_key_team_change,
 )
 from litellm.proxy.proxy_server import app
 
@@ -1033,3 +1034,60 @@ async def test_unblock_key_invalid_key_format(monkeypatch):
 
     assert exc_info.value.code == "400"
     assert "Invalid key format" in str(exc_info.value.message)
+
+
+def test_validate_key_team_change_with_member_permissions():
+    """
+    Test validate_key_team_change function with team member permissions.
+    
+    This test covers the new logic that allows team members with specific
+    permissions to update keys, not just team admins.
+    """
+    from unittest.mock import MagicMock, patch
+
+    from litellm.proxy._types import KeyManagementRoutes
+
+    # Create mock objects
+    mock_key = MagicMock()
+    mock_key.user_id = "test-user-123"
+    mock_key.models = ["gpt-4"]
+    mock_key.tpm_limit = None
+    mock_key.rpm_limit = None
+    
+    mock_team = MagicMock()
+    mock_team.team_id = "test-team-456" 
+    mock_team.members_with_roles = []
+    mock_team.tpm_limit = None
+    mock_team.rpm_limit = None
+    
+    mock_change_initiator = MagicMock()
+    mock_change_initiator.user_id = "test-user-123"
+    
+    mock_router = MagicMock()
+    
+    # Mock the member object returned by _get_user_in_team
+    mock_member_object = MagicMock()
+    
+    with patch('litellm.proxy.management_endpoints.key_management_endpoints.can_team_access_model'):
+        with patch('litellm.proxy.management_endpoints.key_management_endpoints._get_user_in_team') as mock_get_user:
+            with patch('litellm.proxy.management_endpoints.key_management_endpoints._is_user_team_admin') as mock_is_admin:
+                with patch('litellm.proxy.management_endpoints.key_management_endpoints.TeamMemberPermissionChecks.does_team_member_have_permissions_for_endpoint') as mock_has_perms:
+                    
+                    mock_get_user.return_value = mock_member_object
+                    mock_is_admin.return_value = False
+                    mock_has_perms.return_value = True
+                    
+                    # This should not raise an exception due to member permissions
+                    validate_key_team_change(
+                        key=mock_key,
+                        team=mock_team,
+                        change_initiated_by=mock_change_initiator,
+                        llm_router=mock_router
+                    )
+                    
+                    # Verify the permission check was called with correct parameters
+                    mock_has_perms.assert_called_once_with(
+                        team_member_object=mock_member_object,
+                        team_table=mock_team,
+                        route=KeyManagementRoutes.KEY_UPDATE.value
+                    )

tests/test_litellm/proxy/management_endpoints/test_mcp_management_endpoints.py

@@ -22,7 +22,7 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.types.mcp import MCPAuth
-from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer
+from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
 
 def generate_mock_mcp_server_db_record(
@@ -63,10 +63,10 @@ def generate_mock_mcp_server_config_record(
         url=url,
         transport=MCPTransport.http if transport == "http" else MCPTransport.sse,
         auth_type=MCPAuth.api_key if auth_type == "api_key" else None,
-        mcp_info=MCPInfo(
-            server_name=name,
-            description="Config server description",
-        ),
+        mcp_info={
+            "server_name": name,
+            "description": "Config server description",
+        },
     )
 
 

tests/test_litellm/proxy/management_endpoints/test_ui_sso.py

@@ -543,6 +543,7 @@ async def test_get_user_info_from_db():
         assert mock_get_user_object.call_args.kwargs["user_id"] == "krrishd"
 
 
+@pytest.mark.asyncio
 async def test_get_user_info_from_db_alternate_user_id():
     from litellm.proxy.management_endpoints.ui_sso import get_user_info_from_db
 
@@ -1275,7 +1276,7 @@ class TestCLIKeyRegenerationFlow:
             )
             
             # Assert
-            mock_regenerate.assert_called_once_with(existing_key, new_key)
+            mock_regenerate.assert_called_once_with(existing_key=existing_key, new_key=new_key, user_id=None)
             assert result.status_code == 200
             assert "Success" in result.body.decode()
 
@@ -1303,7 +1304,7 @@ class TestCLIKeyRegenerationFlow:
             )
             
             # Assert
-            mock_create.assert_called_once_with(new_key)
+            mock_create.assert_called_once_with(key=new_key, user_id=None)
             assert result.status_code == 200
             assert "Success" in result.body.decode()
 
@@ -1320,8 +1321,17 @@ class TestCLIKeyRegenerationFlow:
         # CLI state
         cli_state = f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:sk-new-session-key-456"
         
-        # Mock the CLI callback
-        with patch("litellm.proxy.management_endpoints.ui_sso.cli_sso_callback") as mock_cli_callback:
+        # Mock the CLI callback and required proxy server components
+        mock_result = {"user_id": "test-user", "email": "test@example.com"}
+        
+        with patch("litellm.proxy.management_endpoints.ui_sso.cli_sso_callback") as mock_cli_callback, \
+             patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), \
+             patch("litellm.proxy.proxy_server.master_key", "test-master-key"), \
+             patch("litellm.proxy.proxy_server.general_settings", {}), \
+             patch("litellm.proxy.proxy_server.jwt_handler", MagicMock()), \
+             patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), \
+             patch.dict(os.environ, {"GOOGLE_CLIENT_ID": "test-google-id"}, clear=True), \
+             patch("litellm.proxy.management_endpoints.ui_sso.GoogleSSOHandler.get_google_callback_response", return_value=mock_result):
             mock_cli_callback.return_value = MagicMock()
             
             # Act
@@ -1329,9 +1339,10 @@ class TestCLIKeyRegenerationFlow:
             
             # Assert
             mock_cli_callback.assert_called_once_with(
-                mock_request,
+                request=mock_request,
                 key="sk-new-session-key-456",
-                existing_key="sk-existing-cli-key-123"
+                existing_key="sk-existing-cli-key-123",
+                result=mock_result
             )
 
     def test_get_redirect_url_preserves_existing_key(self):