scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-30 01:52:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 114

Merge branch 'BerriAI:main' into feat/sambanova-mantainance

Browse source
This commit is contained in:
Jorge Piedrahita Ortiz 2026-02-05 11:53:16 -05:00 • committed by GitHub
commit 4aa87755a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
229 changed files with 16316 additions and 1746 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
.circleci/config.yml
View file

@ -1255,7 +1255,15 @@ jobs:
ls
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
# Subdirectories with dedicated jobs (maintain this list as new jobs are added)
IGNORE_DIRS=(
"tests/llm_translation/realtime"
)
IGNORE_ARGS=""
for dir in "${IGNORE_DIRS[@]}"; do
IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
done
python -m pytest -vv tests/llm_translation $IGNORE_ARGS --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -1271,6 +1279,54 @@ jobs:
paths:
- llm_translation_coverage.xml
- llm_translation_coverage
realtime_translation_testing:
docker:
- image: cimg/python:3.11
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-cov==5.0.0"
pip install "pytest-asyncio==0.21.1"
pip install "respx==0.22.0"
pip install "pytest-xdist==3.6.1"
pip install "pytest-timeout==2.2.0"
pip install "websockets"
# Run pytest and generate JUnit XML report
- run:
name: Run realtime tests
command: |
pwd
ls
# Add --timeout to kill hanging tests after 120s (2 min)
# Add --durations=20 to show 20 slowest tests for debugging
python -m pytest -vv tests/llm_translation/realtime --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread
no_output_timeout: 120m
- run:
name: Rename the coverage files
command: |
mv coverage.xml realtime_translation_coverage.xml
mv .coverage realtime_translation_coverage
# Store test results
- store_test_results:
path: test-results
- persist_to_workspace:
root: .
paths:
- realtime_translation_coverage.xml
- realtime_translation_coverage
mcp_testing:
docker:
- image: cimg/python:3.11
@ -3532,7 +3588,7 @@ jobs:
python -m venv venv
. venv/bin/activate
pip install coverage
coverage combine llm_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage combine llm_translation_coverage realtime_translation_coverage llm_responses_api_coverage ocr_coverage search_coverage mcp_coverage logging_coverage audio_coverage litellm_router_coverage litellm_router_unit_coverage local_testing_part1_coverage local_testing_part2_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_part1_coverage litellm_proxy_unit_tests_part2_coverage image_gen_coverage pass_through_unit_tests_coverage batches_coverage litellm_security_tests_coverage guardrails_coverage litellm_mapped_tests_coverage
coverage xml
- codecov/upload:
file: ./coverage.xml
@ -4196,6 +4252,12 @@ workflows:
only:
- main
- /litellm_.*/
- realtime_translation_testing:
filters:
branches:
only:
- main
- /litellm_.*/
- mcp_testing:
filters:
branches:
@ -4307,6 +4369,7 @@ workflows:
- upload-coverage:
requires:
- llm_translation_testing
- realtime_translation_testing
- mcp_testing
- google_generate_content_endpoint_testing
- guardrails_testing
@ -4384,6 +4447,7 @@ workflows:
- e2e_openai_endpoints
- test_bad_database_url
- llm_translation_testing
- realtime_translation_testing
- mcp_testing
- google_generate_content_endpoint_testing
- llm_responses_api_testing

65
Makefile
View file

@ -1,7 +1,10 @@
# LiteLLM Makefile
# Simple Makefile for running tests and basic development tasks
.PHONY: help test test-unit test-integration test-unit-helm lint format install-dev install-proxy-dev install-test-deps install-helm-unittest check-circular-imports check-import-safety
.PHONY: help test test-unit test-integration test-unit-helm \
info lint lint-dev format \
install-dev install-proxy-dev install-test-deps \
install-helm-unittest check-circular-imports check-import-safety
# Default target
help:
@ -25,6 +28,13 @@ help:
@echo " make test-integration - Run integration tests"
@echo " make test-unit-helm - Run helm unit tests"
# Keep PIP simple for edge cases:
PIP := $(shell command -v pip > /dev/null 2>&1 && echo "pip" || echo "python3 -m pip")
# Show info
info:
@echo "PIP: $(PIP)"
# Installation targets
install-dev:
poetry install --with dev
@ -34,19 +44,19 @@ install-proxy-dev:
# CI-compatible installations (matches GitHub workflows exactly)
install-dev-ci:
pip install openai==2.8.0
$(PIP) install openai==2.8.0
poetry install --with dev
pip install openai==2.8.0
$(PIP) install openai==2.8.0
install-proxy-dev-ci:
poetry install --with dev,proxy-dev --extras proxy
pip install openai==2.8.0
$(PIP) install openai==2.8.0
install-test-deps: install-proxy-dev
poetry run pip install "pytest-retry==1.6.3"
poetry run pip install pytest-xdist
poetry run pip install openapi-core
cd enterprise && poetry run pip install -e . && cd ..
poetry run $(PIP) install "pytest-retry==1.6.3"
poetry run $(PIP) install pytest-xdist
poetry run $(PIP) install openapi-core
cd enterprise && poetry run $(PIP) install -e . && cd ..
install-helm-unittest:
helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
@ -62,8 +72,40 @@ format-check: install-dev
lint-ruff: install-dev
cd litellm && poetry run ruff check . && cd ..
# faster linter for developing ...
# inspiration from:
# https://github.com/astral-sh/ruff/discussions/10977
# https://github.com/astral-sh/ruff/discussions/4049
lint-format-changed: install-dev
@git diff origin/main --unified=0 --no-color -- '*.py' | \
perl -ne '\
if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
$$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
print "$$file:$$start:1-$$end:999\n"; \
}' | \
while read range; do \
file="$${range%%:*}"; \
lines="$${range#*:}"; \
echo "Formatting $$file (lines $$lines)"; \
poetry run ruff format --range "$$lines" "$$file"; \
done
lint-ruff-dev: install-dev
@tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
cd litellm && \
(poetry run ruff check . --output-format=pylint || true) > "$$tmpfile" && \
poetry run diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
cd .. ; \
rm -f "$$tmpfile"
lint-ruff-FULL-dev: install-dev
@files=$$(git diff --name-only origin/main -- '*.py'); \
if [ -n "$$files" ]; then echo "$$files" | xargs poetry run ruff check; \
else echo "No changed .py files to check."; fi
lint-mypy: install-dev
poetry run pip install types-requests types-setuptools types-redis types-PyYAML
poetry run $(PIP) install types-requests types-setuptools types-redis types-PyYAML
cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
lint-black: format-check
@ -72,11 +114,14 @@ check-circular-imports: install-dev
cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
check-import-safety: install-dev
poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
@poetry run python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
# Combined linting (matches test-linting.yml workflow)
lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety
# Testing targets
test:
poetry run pytest tests/

114
cookbook/livekit_agent_sdk/README.md Normal file
View file

@ -0,0 +1,114 @@
# LiveKit Voice Agent with LiteLLM Gateway
Simple example showing how to use LiveKit's xAI realtime plugin with LiteLLM as a proxy. This lets you switch between xAI, OpenAI, and Azure realtime APIs without changing your code.
## Quick Start
### 1. Install dependencies
```bash
pip install livekit-agents[xai] websockets
```
### 2. Start LiteLLM proxy
```bash
# With xAI
export XAI_API_KEY="your-xai-key"
litellm --config config.yaml --port 4000
```
### 3. Run the voice agent
```bash
python main.py
```
Type your message and get a voice response from Grok!
## Configuration
Set these environment variables if needed:
```bash
export LITELLM_PROXY_URL="http://localhost:4000"
export LITELLM_API_KEY="sk-1234"
export LITELLM_MODEL="grok-voice-agent"
```
Or use the defaults - connects to `http://localhost:4000` by default.
## Example Config File
Create a `config.yaml` with your realtime models:
```yaml
model_list:
- model_name: grok-voice-agent
litellm_params:
model: xai/grok-2-vision-1212
api_key: os.environ/XAI_API_KEY
model_info:
mode: realtime
- model_name: openai-voice-agent
litellm_params:
model: gpt-4o-realtime-preview
api_key: os.environ/OPENAI_API_KEY
model_info:
mode: realtime
general_settings:
master_key: sk-1234
```
Then start: `litellm --config config.yaml --port 4000`
## How It Works
LiveKit's xAI plugin connects through LiteLLM proxy by setting `base_url`:
```python
from livekit.plugins import xai
model = xai.realtime.RealtimeModel(
voice="ara",
api_key="sk-1234", # LiteLLM proxy key
base_url="http://localhost:4000", # Point to LiteLLM
)
```
## Switching Providers
Just change the model in your config - no code changes needed:
**xAI Grok:**
```yaml
model: xai/grok-2-vision-1212
```
**OpenAI:**
```yaml
model: gpt-4o-realtime-preview
```
**Azure OpenAI:**
```yaml
model: azure/gpt-4o-realtime-preview
api_base: https://your-endpoint.openai.azure.com/
```
## Why Use LiteLLM?
- ✅ **Switch providers** without changing agent code
- ✅ **Cost tracking** across all voice sessions
- ✅ **Rate limiting** and budgets
- ✅ **Load balancing** across multiple API keys
- ✅ **Fallbacks** to backup models
## Learn More
- [LiveKit xAI Realtime Tutorial](/docs/tutorials/livekit_xai_realtime)
- [xAI Realtime Docs](/docs/providers/xai_realtime)
- [LiveKit Agents Documentation](https://docs.livekit.io/agents/)
- [LiteLLM Realtime API](/docs/realtime)

21
cookbook/livekit_agent_sdk/config.example.yaml Normal file
View file

@ -0,0 +1,21 @@
model_list:
- model_name: grok-voice-agent
litellm_params:
model: xai/grok-2-vision-1212
api_key: os.environ/XAI_API_KEY
model_info:
mode: realtime
- model_name: openai-voice-agent
litellm_params:
model: gpt-4o-realtime-preview
api_key: os.environ/OPENAI_API_KEY
model_info:
mode: realtime
litellm_settings:
drop_params: True
telemetry: False
general_settings:
master_key: sk-1234 # Change this to a secure key

112
cookbook/livekit_agent_sdk/main.py Normal file
View file

@ -0,0 +1,112 @@
"""
Simple xAI Voice Agent using LiveKit SDK with LiteLLM Gateway
This example shows how to use LiveKit's xAI realtime plugin through LiteLLM proxy.
LiteLLM acts as a unified interface, allowing you to switch between xAI, OpenAI,
and Azure realtime APIs without changing your agent code.
"""
import asyncio
import json
import os
import websockets
# Configuration
PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
MODEL = os.getenv("LITELLM_MODEL", "grok-voice-agent")
async def run_voice_agent():
"""
Simple voice agent that:
1. Connects to xAI realtime API through LiteLLM proxy
2. Sends a user message
3. Streams back the response
"""
url = f"ws://{PROXY_URL.replace('http://', '').replace('https://', '')}/v1/realtime?model={MODEL}"
headers = {"Authorization": f"Bearer {API_KEY}"}
print(f"🎙️ Connecting to voice agent...")
print(f" Model: {MODEL}")
print(f" Proxy: {PROXY_URL}")
print()
async with websockets.connect(url, additional_headers=headers) as ws:
# Receive initial connection event
initial = json.loads(await ws.recv())
print(f"✅ Connected! Event: {initial['type']}\n")
# Get user input
user_message = input("💬 Your message: ").strip()
if not user_message:
user_message = "Tell me a fun fact about AI!"
print(f"\n🤖 Sending to {MODEL}...\n")
# Send user message
await ws.send(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": user_message}]
}
}))
# Request response
await ws.send(json.dumps({
"type": "response.create",
"response": {"modalities": ["text", "audio"]}
}))
# Stream response
print("🎤 Response: ", end='', flush=True)
transcript = []
try:
while True:
msg = await asyncio.wait_for(ws.recv(), timeout=15.0)
event = json.loads(msg)
# Capture transcript deltas
if event['type'] == 'response.output_audio_transcript.delta':
delta = event.get('delta', '')
if delta:
print(delta, end='', flush=True)
transcript.append(delta)
# Done when response completes
elif event['type'] == 'response.done':
break
except asyncio.TimeoutError:
pass
print("\n")
if transcript:
print(f"✅ Complete response: {''.join(transcript)}")
await ws.close()
def main():
"""Run the voice agent"""
print("=" * 70)
print("LiveKit xAI Voice Agent via LiteLLM Proxy")
print("=" * 70)
print()
try:
asyncio.run(run_voice_agent())
except KeyboardInterrupt:
print("\n\n👋 Goodbye!")
except Exception as e:
print(f"\n❌ Error: {e}")
print("\nMake sure LiteLLM proxy is running:")
print(f" litellm --config config.yaml --port 4000")
if __name__ == "__main__":
main()

2
cookbook/livekit_agent_sdk/requirements.txt Normal file
View file

@ -0,0 +1,2 @@
livekit-agents[xai]>=1.3.12
websockets>=15.0.1

113
docs/my-website/docs/a2a.md
View file

@ -68,116 +68,9 @@ Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./pr
## Invoking your Agents
Use the [A2A Python SDK](https://pypi.org/project/a2a-sdk) to invoke agents through LiteLLM.
This example shows how to:
1. **List available agents** - Query `/v1/agents` to see which agents your key can access
2. **Select an agent** - Pick an agent from the list
3. **Invoke via A2A** - Use the A2A protocol to send messages to the agent
```python showLineNumbers title="invoke_a2a_agent.py"
from uuid import uuid4
import httpx
import asyncio
from a2a.client import A2ACardResolver, A2AClient
from a2a.types import MessageSendParams, SendMessageRequest
# === CONFIGURE THESE ===
LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL
LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key
# =======================
async def main():
headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
async with httpx.AsyncClient(headers=headers) as client:
# Step 1: List available agents
response = await client.get(f"{LITELLM_BASE_URL}/v1/agents")
agents = response.json()
print("Available agents:")
for agent in agents:
print(f" - {agent['agent_name']} (ID: {agent['agent_id']})")
if not agents:
print("No agents available for this key")
return
# Step 2: Select an agent and invoke it
selected_agent = agents[0]
agent_id = selected_agent["agent_id"]
agent_name = selected_agent["agent_name"]
print(f"\nInvoking: {agent_name}")
# Step 3: Use A2A protocol to invoke the agent
base_url = f"{LITELLM_BASE_URL}/a2a/{agent_id}"
resolver = A2ACardResolver(httpx_client=client, base_url=base_url)
agent_card = await resolver.get_agent_card()
a2a_client = A2AClient(httpx_client=client, agent_card=agent_card)
request = SendMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Hello, what can you do?"}],
"messageId": uuid4().hex,
}
),
)
response = await a2a_client.send_message(request)
print(f"Response: {response.model_dump(mode='json', exclude_none=True, indent=4)}")
if __name__ == "__main__":
asyncio.run(main())
```
### Streaming Responses
For streaming responses, use `send_message_streaming`:
```python showLineNumbers title="invoke_a2a_agent_streaming.py"
from uuid import uuid4
import httpx
import asyncio
from a2a.client import A2ACardResolver, A2AClient
from a2a.types import MessageSendParams, SendStreamingMessageRequest
# === CONFIGURE THESE ===
LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL
LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key
LITELLM_AGENT_NAME = "ij-local" # Agent name registered in LiteLLM
# =======================
async def main():
base_url = f"{LITELLM_BASE_URL}/a2a/{LITELLM_AGENT_NAME}"
headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
async with httpx.AsyncClient(headers=headers) as httpx_client:
# Resolve agent card and create client
resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
agent_card = await resolver.get_agent_card()
client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
# Send a streaming message
request = SendStreamingMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Hello, what can you do?"}],
"messageId": uuid4().hex,
}
),
)
# Stream the response
async for chunk in client.send_message_streaming(request):
print(chunk.model_dump(mode="json", exclude_none=True))
if __name__ == "__main__":
asyncio.run(main())
```
See the [Invoking A2A Agents](./a2a_invoking_agents) guide to learn how to call your agents using:
- **A2A SDK** - Native A2A protocol with full support for tasks and artifacts
- **OpenAI SDK** - Familiar `/chat/completions` interface with `a2a/` model prefix
## Tracking Agent Logs

280
docs/my-website/docs/a2a_invoking_agents.md Normal file
View file

@ -0,0 +1,280 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Invoking A2A Agents
Learn how to invoke A2A agents through LiteLLM using different methods.
:::tip Deploy Your Own A2A Agent
Want to test with your own agent? Deploy this template A2A agent powered by Google Gemini:
[**shin-bot-litellm/a2a-gemini-agent**](https://github.com/shin-bot-litellm/a2a-gemini-agent) - Simple deployable A2A agent with streaming support
:::
## A2A SDK
Use the [A2A Python SDK](https://pypi.org/project/a2a-sdk) to invoke agents through LiteLLM using the A2A protocol.
### Non-Streaming
This example shows how to:
1. **List available agents** - Query `/v1/agents` to see which agents your key can access
2. **Select an agent** - Pick an agent from the list
3. **Invoke via A2A** - Use the A2A protocol to send messages to the agent
```python showLineNumbers title="invoke_a2a_agent.py"
from uuid import uuid4
import httpx
import asyncio
from a2a.client import A2ACardResolver, A2AClient
from a2a.types import MessageSendParams, SendMessageRequest
# === CONFIGURE THESE ===
LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL
LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key
# =======================
async def main():
headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
async with httpx.AsyncClient(headers=headers) as client:
# Step 1: List available agents
response = await client.get(f"{LITELLM_BASE_URL}/v1/agents")
agents = response.json()
print("Available agents:")
for agent in agents:
print(f" - {agent['agent_name']} (ID: {agent['agent_id']})")
if not agents:
print("No agents available for this key")
return
# Step 2: Select an agent and invoke it
selected_agent = agents[0]
agent_id = selected_agent["agent_id"]
agent_name = selected_agent["agent_name"]
print(f"\nInvoking: {agent_name}")
# Step 3: Use A2A protocol to invoke the agent
base_url = f"{LITELLM_BASE_URL}/a2a/{agent_id}"
resolver = A2ACardResolver(httpx_client=client, base_url=base_url)
agent_card = await resolver.get_agent_card()
a2a_client = A2AClient(httpx_client=client, agent_card=agent_card)
request = SendMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Hello, what can you do?"}],
"messageId": uuid4().hex,
}
),
)
response = await a2a_client.send_message(request)
print(f"Response: {response.model_dump(mode='json', exclude_none=True, indent=4)}")
if __name__ == "__main__":
asyncio.run(main())
```
### Streaming
For streaming responses, use `send_message_streaming`:
```python showLineNumbers title="invoke_a2a_agent_streaming.py"
from uuid import uuid4
import httpx
import asyncio
from a2a.client import A2ACardResolver, A2AClient
from a2a.types import MessageSendParams, SendStreamingMessageRequest
# === CONFIGURE THESE ===
LITELLM_BASE_URL = "http://localhost:4000" # Your LiteLLM proxy URL
LITELLM_VIRTUAL_KEY = "sk-1234" # Your LiteLLM Virtual Key
LITELLM_AGENT_NAME = "ij-local" # Agent name registered in LiteLLM
# =======================
async def main():
base_url = f"{LITELLM_BASE_URL}/a2a/{LITELLM_AGENT_NAME}"
headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
async with httpx.AsyncClient(headers=headers) as httpx_client:
# Resolve agent card and create client
resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
agent_card = await resolver.get_agent_card()
client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
# Send a streaming message
request = SendStreamingMessageRequest(
id=str(uuid4()),
params=MessageSendParams(
message={
"role": "user",
"parts": [{"kind": "text", "text": "Tell me a long story"}],
"messageId": uuid4().hex,
}
),
)
# Stream the response
async for chunk in client.send_message_streaming(request):
print(chunk.model_dump(mode="json", exclude_none=True))
if __name__ == "__main__":
asyncio.run(main())
```
## /chat/completions API (OpenAI SDK)
You can also invoke A2A agents using the familiar OpenAI SDK by using the `a2a/` model prefix.
### Non-Streaming
<Tabs>
<TabItem value="python" label="Python" default>
```python showLineNumbers title="openai_non_streaming.py"
import openai
client = openai.OpenAI(
api_key="sk-1234", # Your LiteLLM Virtual Key
base_url="http://localhost:4000" # Your LiteLLM proxy URL
)
response = client.chat.completions.create(
model="a2a/my-agent", # Use a2a/ prefix with your agent name
messages=[
{"role": "user", "content": "Hello, what can you do?"}
]
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="typescript" label="TypeScript">
```typescript showLineNumbers title="openai_non_streaming.ts"
import OpenAI from 'openai';
const client = new OpenAI({
apiKey: 'sk-1234', // Your LiteLLM Virtual Key
baseURL: 'http://localhost:4000' // Your LiteLLM proxy URL
});
const response = await client.chat.completions.create({
model: 'a2a/my-agent', // Use a2a/ prefix with your agent name
messages: [
{ role: 'user', content: 'Hello, what can you do?' }
]
});
console.log(response.choices[0].message.content);
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="curl_non_streaming.sh"
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "a2a/my-agent",
"messages": [
{"role": "user", "content": "Hello, what can you do?"}
]
}'
```
</TabItem>
</Tabs>
### Streaming
<Tabs>
<TabItem value="python" label="Python" default>
```python showLineNumbers title="openai_streaming.py"
import openai
client = openai.OpenAI(
api_key="sk-1234", # Your LiteLLM Virtual Key
base_url="http://localhost:4000" # Your LiteLLM proxy URL
)
stream = client.chat.completions.create(
model="a2a/my-agent", # Use a2a/ prefix with your agent name
messages=[
{"role": "user", "content": "Tell me a long story"}
],
stream=True
)
for chunk in stream:
if chunk.choices[0].delta.content:
print(chunk.choices[0].delta.content, end="", flush=True)
```
</TabItem>
<TabItem value="typescript" label="TypeScript">
```typescript showLineNumbers title="openai_streaming.ts"
import OpenAI from 'openai';
const client = new OpenAI({
apiKey: 'sk-1234', // Your LiteLLM Virtual Key
baseURL: 'http://localhost:4000' // Your LiteLLM proxy URL
});
const stream = await client.chat.completions.create({
model: 'a2a/my-agent', // Use a2a/ prefix with your agent name
messages: [
{ role: 'user', content: 'Tell me a long story' }
],
stream: true
});
for await (const chunk of stream) {
const content = chunk.choices[0]?.delta?.content;
if (content) {
process.stdout.write(content);
}
}
```
</TabItem>
<TabItem value="curl" label="cURL">
```bash showLineNumbers title="curl_streaming.sh"
curl -X POST http://localhost:4000/v1/chat/completions \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "a2a/my-agent",
"messages": [
{"role": "user", "content": "Tell me a long story"}
],
"stream": true
}'
```
</TabItem>
</Tabs>
## Key Differences
| Method | Use Case | Advantages |
|--------|----------|------------|
| **A2A SDK** | Native A2A protocol integration | • Full A2A protocol support<br/>• Access to task states and artifacts<br/>• Context management |
| **OpenAI SDK** | Familiar OpenAI-style interface | • Drop-in replacement for OpenAI calls<br/>• Easier migration from LLM to agent workflows<br/>• Works with existing OpenAI tooling |
:::tip Model Prefix
When using the OpenAI SDK, always prefix your agent name with `a2a/` (e.g., `a2a/my-agent`) to route requests to the A2A agent instead of an LLM provider.
:::

7
docs/my-website/docs/adding_provider/simple_guardrail_tutorial.md
View file

@ -101,12 +101,11 @@ model_list:
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
api_key: os.environ/OPENAI_API_KEY
litellm_settings:
guardrails:
guardrails:
- guardrail_name: my_guardrail
litellm_params:
litellm_params:
guardrail: my_guardrail
mode: during_call
api_key: os.environ/MY_GUARDRAIL_API_KEY

60
docs/my-website/docs/observability/langfuse_integration.md
View file

@ -215,6 +215,66 @@ The following parameters can be updated on a continuation of a trace by passing
Any other key value pairs passed into the metadata not listed in the above spec for a `litellm` completion will be added as a metadata key value pair for the generation.
#### Multiple Langfuse Projects (Per-Request Credentials)
You can send traces to different Langfuse projects per request by passing credentials directly to `completion()` or `acompletion()`. This works alongside (or instead of) the global env vars and is useful when different teams or business processes use different Langfuse projects.
Pass **`langfuse_public_key`**, **`langfuse_secret_key`** (or **`langfuse_secret`**), and optionally **`langfuse_host`** as keyword arguments:
```python
import litellm
from litellm import completion
# Optional: set a default via env for requests that don't pass credentials
# os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-default..."
# os.environ["LANGFUSE_SECRET_KEY"] = "sk-default..."
litellm.success_callback = ["langfuse"]
litellm.failure_callback = ["langfuse"]
# Request 1 → Langfuse Project A
response_a = completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello from team A"}],
langfuse_public_key="pk-lf-project-a...",
langfuse_secret_key="sk-lf-project-a...",
langfuse_host="https://us.cloud.langfuse.com", # optional
)
# Request 2 → Langfuse Project B (different project)
response_b = completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hello from team B"}],
langfuse_public_key="pk-lf-project-b...",
langfuse_secret_key="sk-lf-project-b...",
langfuse_host="https://eu.cloud.langfuse.com", # optional, can differ per project
)
```
Async usage with per-request credentials:
```python
import litellm
from litellm import acompletion
litellm.success_callback = ["langfuse"]
litellm.failure_callback = ["langfuse"]
response = await acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hi"}],
langfuse_public_key="pk-lf-...",
langfuse_secret_key="sk-lf-...",
langfuse_host="https://us.cloud.langfuse.com", # optional
)
```
- **`langfuse_public_key`** – Langfuse project public key (required for per-request override).
- **`langfuse_secret_key`** or **`langfuse_secret`** – Langfuse secret key (either name is accepted).
- **`langfuse_host`** – Langfuse host URL (e.g. `https://us.cloud.langfuse.com`); optional, defaults to env or Langfuse cloud.
When these are passed, that request uses this project (and host) for the Langfuse callback; when omitted, the callback uses the global Langfuse client (from env vars if set). LiteLLM caches a Langfuse client per credential set to avoid creating a new client on every request.
#### Disable Logging - Specific Calls
To disable logging for specific calls use the `no-log` flag.

35
docs/my-website/docs/providers/github_copilot.md
View file

@ -35,11 +35,10 @@ from litellm import completion
response = completion(
model="github_copilot/gpt-4",
messages=[{"role": "user", "content": "Write a Python function to calculate fibonacci numbers"}],
extra_headers={
"editor-version": "vscode/1.85.1",
"Copilot-Integration-Id": "vscode-chat"
}
messages=[
{"role": "system", "content": "You are a helpful coding assistant"},
{"role": "user", "content": "Write a Python function to calculate fibonacci numbers"}
]
)
print(response)
```
@ -50,11 +49,7 @@ from litellm import completion
stream = completion(
model="github_copilot/gpt-4",
messages=[{"role": "user", "content": "Explain async/await in Python"}],
stream=True,
extra_headers={
"editor-version": "vscode/1.85.1",
"Copilot-Integration-Id": "vscode-chat"
}
stream=True
)
for chunk in stream:
@ -134,11 +129,7 @@ client = OpenAI(
# Non-streaming response
response = client.chat.completions.create(
model="github_copilot/gpt-4",
messages=[{"role": "user", "content": "How do I optimize this SQL query?"}],
extra_headers={
"editor-version": "vscode/1.85.1",
"Copilot-Integration-Id": "vscode-chat"
}
messages=[{"role": "user", "content": "How do I optimize this SQL query?"}]
)
print(response.choices[0].message.content)
@ -156,11 +147,7 @@ response = litellm.completion(
model="litellm_proxy/github_copilot/gpt-4",
messages=[{"role": "user", "content": "Review this code for bugs"}],
api_base="http://localhost:4000",
api_key="your-proxy-api-key",
extra_headers={
"editor-version": "vscode/1.85.1",
"Copilot-Integration-Id": "vscode-chat"
}
api_key="your-proxy-api-key"
)
print(response.choices[0].message.content)
@ -174,8 +161,6 @@ print(response.choices[0].message.content)
curl http://localhost:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer your-proxy-api-key" \
-H "editor-version: vscode/1.85.1" \
-H "Copilot-Integration-Id: vscode-chat" \
-d '{
"model": "github_copilot/gpt-4",
"messages": [{"role": "user", "content": "Explain this error message"}]
@ -211,9 +196,11 @@ export GITHUB_COPILOT_API_KEY_FILE="api-key.json"
### Headers
GitHub Copilot supports various editor-specific headers:
LiteLLM automatically injects the required GitHub Copilot headers (simulating VSCode). You don't need to specify them manually.
```python showLineNumbers title="Common Headers"
If you want to override the defaults (e.g., to simulate a different editor), you can use `extra_headers`:
```python showLineNumbers title="Custom Headers (Optional)"
extra_headers = {
"editor-version": "vscode/1.85.1", # Editor version
"editor-plugin-version": "copilot/1.155.0", # Plugin version

308
docs/my-website/docs/providers/xai_realtime.md Normal file
View file

@ -0,0 +1,308 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# xAI Voice Agent (Realtime API)
xAI's Grok Voice Agent provides real-time voice conversation capabilities through WebSocket connections, enabling natural bidirectional audio interactions.
| Feature | Description | Comments |
| --- | --- | --- |
| LiteLLM AI Gateway | ✅ | |
| LiteLLM Python SDK | ✅ | Full support via `litellm.realtime()` |
## Quick Start
### Supported Model
| Model | Context | Features |
|-------|---------|----------|
| `xai/grok-4-1-fast-non-reasoning` | 2M tokens | Voice conversation, Function calling, Vision, Audio, Web search, Caching |
**Note:** xAI Realtime API uses the non-reasoning variant for optimal real-time performance.
## Python SDK Usage
### Basic Realtime Connection
```python
import asyncio
from litellm import realtime
async def test_xai_realtime():
"""
Test xAI Grok Voice Agent via LiteLLM SDK
"""
# Initialize realtime connection
ws = await realtime(
model="xai/grok-4-1-fast-non-reasoning",
api_key="your-xai-api-key", # or set XAI_API_KEY env var
)
# Connection established, xAI sends "conversation.created" event
print("Connected to xAI Grok Voice Agent")
# Send a message
await ws.send_text(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{
"type": "input_text",
"text": "Hello! How are you?"
}]
}
}))
# Request a response
await ws.send_text(json.dumps({
"type": "response.create"
}))
# Listen for responses
async for message in ws:
data = json.loads(message)
print(f"Received: {data['type']}")
if data['type'] == 'response.done':
break
await ws.close()
# Run the async function
asyncio.run(test_xai_realtime())
```
### With Audio Input/Output
```python
import asyncio
import json
from litellm import realtime
async def xai_voice_conversation():
"""
Voice conversation with xAI Grok Voice Agent
"""
ws = await realtime(
model="xai/grok-4-1-fast-non-reasoning",
api_key="your-xai-api-key",
)
# Send audio data (base64 encoded PCM16 24kHz)
await ws.send_text(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{
"type": "input_audio",
"audio": "base64_encoded_audio_data_here"
}]
}
}))
# Request response with audio
await ws.send_text(json.dumps({
"type": "response.create",
"response": {
"modalities": ["text", "audio"],
"instructions": "Please respond in a friendly tone."
}
}))
# Process streaming audio response
async for message in ws:
data = json.loads(message)
if data['type'] == 'response.audio.delta':
# Handle audio chunks
audio_chunk = data['delta']
# Process audio_chunk (play it, save it, etc.)
elif data['type'] == 'response.done':
break
await ws.close()
asyncio.run(xai_voice_conversation())
```
## LiteLLM Proxy (AI Gateway) Usage
Load balance across multiple xAI deployments or combine with other providers.
### 1. Add Model to Config
```yaml
model_list:
- model_name: grok-voice-agent
litellm_params:
model: xai/grok-4-1-fast-non-reasoning
api_key: os.environ/XAI_API_KEY
model_info:
mode: realtime
# Optional: Add fallback to OpenAI
- model_name: grok-voice-agent
litellm_params:
model: openai/gpt-4o-realtime-preview-2024-10-01
api_key: os.environ/OPENAI_API_KEY
model_info:
mode: realtime
```
### 2. Start Proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### 3. Test Connection
#### Python Client
```python
import asyncio
import websockets
import json
async def test_proxy():
url = "ws://0.0.0.0:4000/v1/realtime?model=grok-voice-agent"
async with websockets.connect(
url,
extra_headers={
"Authorization": "Bearer sk-1234", # Your LiteLLM proxy key
"OpenAI-Beta": "realtime=v1"
}
) as ws:
# Wait for conversation.created event from xAI
message = await ws.recv()
print(f"Connected: {message}")
# Send a message
await ws.send(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{
"type": "input_text",
"text": "Hello from LiteLLM proxy!"
}]
}
}))
# Request response
await ws.send(json.dumps({
"type": "response.create"
}))
# Listen for response
async for message in ws:
data = json.loads(message)
print(f"Event: {data['type']}")
if data['type'] == 'response.done':
break
asyncio.run(test_proxy())
```
#### Node.js Client
```javascript
// test.js - Run with: node test.js
const WebSocket = require("ws");
const url = "ws://0.0.0.0:4000/v1/realtime?model=grok-voice-agent";
const ws = new WebSocket(url, {
headers: {
"Authorization": "Bearer sk-1234",
"OpenAI-Beta": "realtime=v1",
},
});
ws.on("open", function open() {
console.log("Connected to xAI via LiteLLM proxy");
// Send a message
ws.send(JSON.stringify({
type: "conversation.item.create",
item: {
type: "message",
role: "user",
content: [{
type: "input_text",
text: "What's the weather like?"
}]
}
}));
// Request response
ws.send(JSON.stringify({
type: "response.create",
response: {
modalities: ["text"],
instructions: "Please assist the user."
}
}));
});
ws.on("message", function incoming(message) {
const data = JSON.parse(message.toString());
console.log(`Event: ${data.type}`);
if (data.type === 'response.done') {
ws.close();
}
});
ws.on("error", function handleError(error) {
console.error("Error: ", error);
});
```
## Key Differences from OpenAI
xAI's Grok Voice Agent has some differences from OpenAI's Realtime API:
| Feature | xAI | OpenAI | LiteLLM Handling |
|---------|-----|--------|------------------|
| Initial Event | `conversation.created` | `session.created` | ⚠️ Passed through as-is |
| WebSocket URL | `wss://api.x.ai/v1/realtime` | `wss://api.openai.com/v1/realtime` | ✅ Auto-configured |
| Model | `grok-4-1-fast-non-reasoning` | `gpt-4o-realtime-preview` | ✅ Via model prefix |
| Audio Format | PCM16 24kHz mono | PCM16 24kHz mono | ✅ Compatible |
| Context Window | 2M tokens | 128K tokens | N/A |
**What LiteLLM Handles:**
- ✅ Automatic URL routing to correct provider
- ✅ Authentication headers (no `OpenAI-Beta` header for xAI)
- ✅ WebSocket connection management
- ✅ All other event types are compatible
**What You Need to Handle:**
- ⚠️ Initial event type difference (`conversation.created` vs `session.created`)
**Tip:** Make your client compatible with both event types:
```python
# Handle both providers
if event['type'] in ['session.created', 'conversation.created']:
print("Connection established")
```
## Related Documentation
- [xAI Chat/Text Models](/docs/providers/xai)
- [LiteLLM Realtime API Overview](/docs/realtime)
- [xAI Official Documentation](https://docs.x.ai/docs)
## Support
For issues or questions:
- [LiteLLM GitHub Issues](https://github.com/BerriAI/litellm/issues)
- [xAI Documentation](https://docs.x.ai/docs)

73
docs/my-website/docs/proxy/admin_ui_sso.md
View file

@ -23,26 +23,75 @@ From v1.76.0, SSO is now Free for up to 5 users.
<Tabs>
<TabItem value="okta" label="Okta SSO">
1. Add Okta credentials to your .env
#### Step 1: Create an OIDC Application in Okta
In your Okta Admin Console, create a new **OIDC Web Application**. See [Okta's guide on creating OIDC app integrations](https://help.okta.com/en-us/content/topics/apps/apps_app_integration_wizard_oidc.htm) for detailed instructions.
When configuring the application:
- **Sign-in redirect URI**: `https://<your-proxy-base-url>/sso/callback`
- **Sign-out redirect URI** (optional): `https://<your-proxy-base-url>`
<Image img={require('../../img/okta_redirect_uri.png')} />
After creating the app, copy your **Client ID** and **Client Secret** from the application's General tab:
<Image img={require('../../img/okta_client_credentials.png')} />
#### Step 2: Assign Users to the Application
Ensure users are assigned to the app in the **Assignments** tab. If Federation Broker Mode is enabled, you may need to disable it to assign users manually.
#### Step 3: Configure Authorization Server Access Policy
:::warning Important
This step is required. Without an Access Policy for your app, users will get a `no_matching_policy` error when attempting to log in.
:::
1. Go to **Security** → **API**
<Image img={require('../../img/okta_security_api.png')} />
2. Select the **default** authorization server (or your custom one)
<Image img={require('../../img/okta_authorization_server.png')} />
3. Click on **Access Policies** tab, create a new policy assigned to your LiteLLM app
4. Add a rule that allows the **Authorization Code** grant type
<Image img={require('../../img/okta_access_policies.png')} />
See [Okta's Access Policy documentation](https://help.okta.com/en-us/content/topics/security/api-access-management/access-policies.htm) for more details.
#### Step 4: Configure LiteLLM Environment Variables
```bash
GENERIC_CLIENT_ID = "<your-okta-client-id>"
GENERIC_CLIENT_SECRET = "<your-okta-client-secret>"
GENERIC_AUTHORIZATION_ENDPOINT = "<your-okta-domain>/authorize" # https://dev-2kqkcd6lx6kdkuzt.us.auth0.com/authorize
GENERIC_TOKEN_ENDPOINT = "<your-okta-domain>/token" # https://dev-2kqkcd6lx6kdkuzt.us.auth0.com/oauth/token
GENERIC_USERINFO_ENDPOINT = "<your-okta-domain>/userinfo" # https://dev-2kqkcd6lx6kdkuzt.us.auth0.com/userinfo
GENERIC_CLIENT_STATE = "random-string" # [OPTIONAL] REQUIRED BY OKTA, if not set random state value is generated
GENERIC_SSO_HEADERS = "Content-Type=application/json, X-Custom-Header=custom-value" # [OPTIONAL] Comma-separated list of additional headers to add to the request - e.g. Content-Type=application/json, etc.
GENERIC_CLIENT_ID="<your-client-id>"
GENERIC_CLIENT_SECRET="<your-client-secret>"
GENERIC_AUTHORIZATION_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/authorize"
GENERIC_TOKEN_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/token"
GENERIC_USERINFO_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/userinfo"
GENERIC_CLIENT_STATE="random-string"
PROXY_BASE_URL="https://<your-proxy-base-url>"
```
You can get your domain specific auth/token/userinfo endpoints at `<YOUR-OKTA-DOMAIN>/.well-known/openid-configuration`
:::tip
You can find all OAuth endpoints at `https://<your-okta-domain>/.well-known/openid-configuration`
:::
2. Add proxy url as callback_url on Okta
#### Step 5: Test the SSO Flow
On Okta, add the 'callback_url' as `<proxy_base_url>/sso/callback`
1. Start your LiteLLM proxy
2. Navigate to `https://<your-proxy-base-url>/ui`
3. Click the SSO login button
4. Authenticate with Okta and verify you're redirected back to LiteLLM
#### Troubleshooting
<Image img={require('../../img/okta_callback_url.png')} />
| Error | Cause | Solution |
|-------|-------|----------|
| `redirect_uri` error | Redirect URI not configured | Add `<proxy_base_url>/sso/callback` to Sign-in redirect URIs in Okta |
| `access_denied` | User not assigned to app | Assign the user in the Assignments tab |
| `no_matching_policy` | Missing Access Policy | Create an Access Policy in the Authorization Server (see Step 3) |
</TabItem>
<TabItem value="google" label="Google SSO">

391
docs/my-website/docs/proxy/cli.md
View file

@ -1,7 +1,10 @@
# CLI Arguments
Cli arguments, --host, --port, --num_workers
## --host
This page documents all command-line interface (CLI) arguments available for the LiteLLM proxy server.
## Server Configuration
### --host
- **Default:** `'0.0.0.0'`
- The host for the server to listen on.
- **Usage:**
@ -14,7 +17,7 @@ Cli arguments, --host, --port, --num_workers
litellm
```
## --port
### --port
- **Default:** `4000`
- The port to bind the server to.
- **Usage:**
@ -27,9 +30,9 @@ Cli arguments, --host, --port, --num_workers
litellm
```
## --num_workers
- **Default:** `1`
- The number of uvicorn workers to spin up.
### --num_workers
- **Default:** Number of logical CPUs in the system, or `4` if that cannot be determined
- The number of uvicorn / gunicorn workers to spin up.
- **Usage:**
```shell
litellm --num_workers 4
@ -40,55 +43,273 @@ Cli arguments, --host, --port, --num_workers
litellm
```
## --api_base
### --config
- **Short form:** `-c`
- **Default:** `None`
- The API base for the model litellm should call.
- Path to the proxy configuration file (e.g., config.yaml).
- **Usage:**
```shell
litellm --config path/to/config.yaml
```
### --log_config
- **Default:** `None`
- **Type:** `str`
- Path to the logging configuration file for uvicorn.
- **Usage:**
```shell
litellm --log_config path/to/log_config.conf
```
### --keepalive_timeout
- **Default:** `None`
- **Type:** `int`
- Set the uvicorn keepalive timeout in seconds (uvicorn timeout_keep_alive parameter).
- **Usage:**
```shell
litellm --keepalive_timeout 30
```
- **Usage - set Environment Variable:** `KEEPALIVE_TIMEOUT`
```shell
export KEEPALIVE_TIMEOUT=30
litellm
```
### --max_requests_before_restart
- **Default:** `None`
- **Type:** `int`
- Restart worker after this many requests. This is useful for mitigating memory growth over time.
- For uvicorn: maps to `limit_max_requests`
- For gunicorn: maps to `max_requests`
- **Usage:**
```shell
litellm --max_requests_before_restart 10000
```
- **Usage - set Environment Variable:** `MAX_REQUESTS_BEFORE_RESTART`
```shell
export MAX_REQUESTS_BEFORE_RESTART=10000
litellm
```
## Server Backend Options
### --run_gunicorn
- **Default:** `False`
- **Type:** `bool` (Flag)
- Starts proxy via gunicorn instead of uvicorn. Better for managing multiple workers in production.
- **Usage:**
```shell
litellm --run_gunicorn
```
### --run_hypercorn
- **Default:** `False`
- **Type:** `bool` (Flag)
- Starts proxy via hypercorn instead of uvicorn. Supports HTTP/2.
- **Usage:**
```shell
litellm --run_hypercorn
```
### --skip_server_startup
- **Default:** `False`
- **Type:** `bool` (Flag)
- Skip starting the server after setup (useful for database migrations only).
- **Usage:**
```shell
litellm --skip_server_startup
```
## SSL/TLS Configuration
### --ssl_keyfile_path
- **Default:** `None`
- **Type:** `str`
- Path to the SSL keyfile. Use this when you want to provide SSL certificate when starting proxy.
- **Usage:**
```shell
litellm --ssl_keyfile_path /path/to/key.pem --ssl_certfile_path /path/to/cert.pem
```
- **Usage - set Environment Variable:** `SSL_KEYFILE_PATH`
```shell
export SSL_KEYFILE_PATH=/path/to/key.pem
litellm
```
### --ssl_certfile_path
- **Default:** `None`
- **Type:** `str`
- Path to the SSL certfile. Use this when you want to provide SSL certificate when starting proxy.
- **Usage:**
```shell
litellm --ssl_certfile_path /path/to/cert.pem --ssl_keyfile_path /path/to/key.pem
```
- **Usage - set Environment Variable:** `SSL_CERTFILE_PATH`
```shell
export SSL_CERTFILE_PATH=/path/to/cert.pem
litellm
```
### --ciphers
- **Default:** `None`
- **Type:** `str`
- Ciphers to use for the SSL setup. Only used with `--run_hypercorn`.
- **Usage:**
```shell
litellm --run_hypercorn --ssl_keyfile_path /path/to/key.pem --ssl_certfile_path /path/to/cert.pem --ciphers "ECDHE+AESGCM"
```
## Model Configuration
### --model or -m
- **Default:** `None`
- The model name to pass to LiteLLM.
- **Usage:**
```shell
litellm --model gpt-3.5-turbo
```
### --alias
- **Default:** `None`
- An alias for the model, for user-friendly reference. Use this to give a litellm model name (e.g., "huggingface/codellama/CodeLlama-7b-Instruct-hf") a more user-friendly name ("codellama").
- **Usage:**
```shell
litellm --alias my-gpt-model
```
### --api_base
- **Default:** `None`
- The API base for the model LiteLLM should call.
- **Usage:**
```shell
litellm --model huggingface/tinyllama --api_base https://k58ory32yinf1ly0.us-east-1.aws.endpoints.huggingface.cloud
```
## --api_version
- **Default:** `None`
### --api_version
- **Default:** `2024-07-01-preview`
- For Azure services, specify the API version.
- **Usage:**
```shell
litellm --model azure/gpt-deployment --api_version 2023-08-01 --api_base https://<your api base>"
```
## --model or -m
### --headers
- **Default:** `None`
- The model name to pass to Litellm.
- Headers for the API call (as JSON string).
- **Usage:**
```shell
litellm --model gpt-3.5-turbo
litellm --model my-model --headers '{"Authorization": "Bearer token"}'
```
## --test
- **Type:** `bool` (Flag)
- Proxy chat completions URL to make a test request.
- **Usage:**
```shell
litellm --test
```
## --health
- **Type:** `bool` (Flag)
- Runs a health check on all models in config.yaml
- **Usage:**
```shell
litellm --health
```
## --alias
### --add_key
- **Default:** `None`
- An alias for the model, for user-friendly reference.
- Add a key to the model configuration.
- **Usage:**
```shell
litellm --alias my-gpt-model
litellm --add_key my-api-key
```
## --debug
### --save
- **Type:** `bool` (Flag)
- Save the model-specific config.
- **Usage:**
```shell
litellm --model gpt-3.5-turbo --save
```
## Model Parameters
### --temperature
- **Default:** `None`
- **Type:** `float`
- Set the temperature for the model.
- **Usage:**
```shell
litellm --temperature 0.7
```
### --max_tokens
- **Default:** `None`
- **Type:** `int`
- Set the maximum number of tokens for the model output.
- **Usage:**
```shell
litellm --max_tokens 50
```
### --request_timeout
- **Default:** `None`
- **Type:** `int`
- Set the timeout in seconds for completion calls.
- **Usage:**
```shell
litellm --request_timeout 300
```
### --max_budget
- **Default:** `None`
- **Type:** `float`
- Set max budget for API calls. Works for hosted models like OpenAI, TogetherAI, Anthropic, etc.
- **Usage:**
```shell
litellm --max_budget 100.0
```
### --drop_params
- **Type:** `bool` (Flag)
- Drop any unmapped params.
- **Usage:**
```shell
litellm --drop_params
```
### --add_function_to_prompt
- **Type:** `bool` (Flag)
- If a function passed but unsupported, pass it as a part of the prompt.
- **Usage:**
```shell
litellm --add_function_to_prompt
```
## Database Configuration
### --iam_token_db_auth
- **Default:** `False`
- **Type:** `bool` (Flag)
- Connects to an RDS database using IAM token authentication instead of a password. This is useful for AWS RDS instances that are configured to use IAM database authentication.
- When enabled, LiteLLM will generate an IAM authentication token to connect to the database.
- **Required Environment Variables:**
- `DATABASE_HOST` - The RDS database host
- `DATABASE_PORT` - The database port
- `DATABASE_USER` - The database user
- `DATABASE_NAME` - The database name
- `DATABASE_SCHEMA` (optional) - The database schema
- **Usage:**
```shell
litellm --iam_token_db_auth
```
- **Usage - set Environment Variable:** `IAM_TOKEN_DB_AUTH`
```shell
export IAM_TOKEN_DB_AUTH=True
export DATABASE_HOST=mydb.us-east-1.rds.amazonaws.com
export DATABASE_PORT=5432
export DATABASE_USER=mydbuser
export DATABASE_NAME=mydb
litellm
```
### --use_prisma_db_push
- **Default:** `False`
- **Type:** `bool` (Flag)
- Use `prisma db push` instead of `prisma migrate` for database schema updates. This is useful when you want to quickly sync your database schema without creating migration files.
- **Usage:**
```shell
litellm --use_prisma_db_push
```
## Debugging
### --debug
- **Default:** `False`
- **Type:** `bool` (Flag)
- Enable debugging mode for the input.
@ -102,10 +323,10 @@ Cli arguments, --host, --port, --num_workers
litellm
```
## --detailed_debug
### --detailed_debug
- **Default:** `False`
- **Type:** `bool` (Flag)
- Enable debugging mode for the input.
- Enable detailed debugging mode to view verbose debug logs.
- **Usage:**
```shell
litellm --detailed_debug
@ -116,80 +337,76 @@ Cli arguments, --host, --port, --num_workers
litellm
```
#### --temperature
- **Default:** `None`
- **Type:** `float`
- Set the temperature for the model.
- **Usage:**
```shell
litellm --temperature 0.7
```
## --max_tokens
- **Default:** `None`
- **Type:** `int`
- Set the maximum number of tokens for the model output.
- **Usage:**
```shell
litellm --max_tokens 50
```
## --request_timeout
- **Default:** `6000`
- **Type:** `int`
- Set the timeout in seconds for completion calls.
- **Usage:**
```shell
litellm --request_timeout 300
```
## --drop_params
### --local
- **Default:** `False`
- **Type:** `bool` (Flag)
- Drop any unmapped params.
- For local debugging purposes.
- **Usage:**
```shell
litellm --drop_params
litellm --local
```
## --add_function_to_prompt
## Testing & Health Checks
### --test
- **Type:** `bool` (Flag)
- If a function passed but unsupported, pass it as a part of the prompt.
- Proxy chat completions URL to make a test request to.
- **Usage:**
```shell
litellm --add_function_to_prompt
litellm --test
```
## --config
- Configure Litellm by providing a configuration file path.
### --test_async
- **Default:** `False`
- **Type:** `bool` (Flag)
- Calls async endpoints `/queue/requests` and `/queue/response`.
- **Usage:**
```shell
litellm --config path/to/config.yaml
litellm --test_async
```
## --telemetry
### --num_requests
- **Default:** `10`
- **Type:** `int`
- Number of requests to hit async endpoint with (used with `--test_async`).
- **Usage:**
```shell
litellm --test_async --num_requests 100
```
### --health
- **Type:** `bool` (Flag)
- Runs a health check on all models in config.yaml.
- **Usage:**
```shell
litellm --health
```
## Other Options
### --version
- **Short form:** `-v`
- **Type:** `bool` (Flag)
- Print LiteLLM version and exit.
- **Usage:**
```shell
litellm --version
```
### --telemetry
- **Default:** `True`
- **Type:** `bool`
- Help track usage of this feature.
- Help track usage of this feature. Turn off for privacy.
- **Usage:**
```shell
litellm --telemetry False
```
## --log_config
- **Default:** `None`
- **Type:** `str`
- Specify a log configuration file for uvicorn.
- **Usage:**
```shell
litellm --log_config path/to/log_config.conf
```
## --skip_server_startup
### --use_queue
- **Default:** `False`
- **Type:** `bool` (Flag)
- Skip starting the server after setup (useful for DB migrations only).
- To use celery workers for async endpoints.
- **Usage:**
```shell
litellm --skip_server_startup
```
litellm --use_queue
```

4
docs/my-website/docs/proxy/config_settings.md
View file

@ -94,7 +94,7 @@ litellm_settings:
# /chat/completions, /completions, /embeddings, /audio/transcriptions
mode: default_off # if default_off, you need to opt in to caching on a per call basis
ttl: 600 # ttl for caching
disable_copilot_system_to_assistant: False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
disable_copilot_system_to_assistant: False # DEPRECATED - GitHub Copilot API supports system prompts.
callback_settings:
otel:
@ -197,7 +197,7 @@ router_settings:
| disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. |
| disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). |
| enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
| disable_copilot_system_to_assistant | boolean | If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. Useful for tools (like Claude Code) that send system messages, which Copilot does not support. |
| disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
### general_settings - Reference

278
docs/my-website/docs/proxy/guardrails/custom_code_guardrail.md Normal file
View file

@ -0,0 +1,278 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Custom Code Guardrail
Write custom guardrail logic using Python-like code that runs in a sandboxed environment.
## Quick Start
### 1. Define the guardrail in config
```yaml
model_list:
- model_name: gpt-4
litellm_params:
model: gpt-4
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: block-ssn
litellm_params:
guardrail: custom_code
mode: pre_call
custom_code: |
def apply_guardrail(inputs, request_data, input_type):
for text in inputs["texts"]:
if regex_match(text, r"\d{3}-\d{2}-\d{4}"):
return block("SSN detected")
return allow()
```
### 2. Start proxy
```bash
litellm --config config.yaml
```
### 3. Test
```bash
curl -X POST http://localhost:4000/chat/completions \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "My SSN is 123-45-6789"}],
"guardrails": ["block-ssn"]
}'
```
## Configuration
| Parameter | Type | Required | Description |
|-----------|------|----------|-------------|
| `guardrail` | string | ✅ | Must be `custom_code` |
| `mode` | string | ✅ | When to run: `pre_call`, `post_call`, `during_call` |
| `custom_code` | string | ✅ | Python-like code with `apply_guardrail` function |
| `default_on` | bool | ❌ | Run on all requests (default: `false`) |
## Writing Custom Code
### Function Signature
Your code must define an `apply_guardrail` function:
```python
def apply_guardrail(inputs, request_data, input_type):
# inputs: see table below
# request_data: {"model": "...", "user_id": "...", "team_id": "...", "metadata": {...}}
# input_type: "request" or "response"
return allow() # or block() or modify()
```
### `inputs` Parameter
| Field | Type | Description |
|-------|------|-------------|
| `texts` | `List[str]` | Extracted text from the request/response |
| `images` | `List[str]` | Extracted images (for image guardrails) |
| `tools` | `List[dict]` | Tools sent to the LLM |
| `tool_calls` | `List[dict]` | Tool calls returned from the LLM |
| `structured_messages` | `List[dict]` | Full messages with role info (system/user/assistant) |
| `model` | `str` | The model being used |
### `request_data` Parameter
| Field | Type | Description |
|-------|------|-------------|
| `model` | `str` | Model name |
| `user_id` | `str` | User ID from API key |
| `team_id` | `str` | Team ID from API key |
| `end_user_id` | `str` | End user ID |
| `metadata` | `dict` | Request metadata |
### Return Values
| Function | Description |
|----------|-------------|
| `allow()` | Let request/response through |
| `block(reason)` | Reject with message |
| `modify(texts=[], images=[], tool_calls=[])` | Transform content |
## Built-in Primitives
### Regex
| Function | Description |
|----------|-------------|
| `regex_match(text, pattern)` | Returns `True` if pattern found |
| `regex_replace(text, pattern, replacement)` | Replace all matches |
| `regex_find_all(text, pattern)` | Return list of matches |
### JSON
| Function | Description |
|----------|-------------|
| `json_parse(text)` | Parse JSON string, returns `None` on error |
| `json_stringify(obj)` | Convert to JSON string |
| `json_schema_valid(obj, schema)` | Validate against JSON schema |
### URL
| Function | Description |
|----------|-------------|
| `extract_urls(text)` | Extract all URLs from text |
| `is_valid_url(url)` | Check if URL is valid |
| `all_urls_valid(text)` | Check all URLs in text are valid |
### Code Detection
| Function | Description |
|----------|-------------|
| `detect_code(text)` | Returns `True` if code detected |
| `detect_code_languages(text)` | Returns list of detected languages |
| `contains_code_language(text, ["sql", "python"])` | Check for specific languages |
### Text Utilities
| Function | Description |
|----------|-------------|
| `contains(text, substring)` | Check if substring exists |
| `contains_any(text, [substr1, substr2])` | Check if any substring exists |
| `word_count(text)` | Count words |
| `char_count(text)` | Count characters |
| `lower(text)` / `upper(text)` / `trim(text)` | String transforms |
## Examples
### Block PII (SSN)
```python
def apply_guardrail(inputs, request_data, input_type):
for text in inputs["texts"]:
if regex_match(text, r"\d{3}-\d{2}-\d{4}"):
return block("SSN detected")
return allow()
```
### Redact Email Addresses
```python
def apply_guardrail(inputs, request_data, input_type):
pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"
modified = []
for text in inputs["texts"]:
modified.append(regex_replace(text, pattern, "[EMAIL REDACTED]"))
return modify(texts=modified)
```
### Block SQL Injection
```python
def apply_guardrail(inputs, request_data, input_type):
if input_type != "request":
return allow()
for text in inputs["texts"]:
if contains_code_language(text, ["sql"]):
return block("SQL code not allowed")
return allow()
```
### Validate JSON Response
```python
def apply_guardrail(inputs, request_data, input_type):
if input_type != "response":
return allow()
schema = {
"type": "object",
"required": ["name", "value"]
}
for text in inputs["texts"]:
obj = json_parse(text)
if obj is None:
return block("Invalid JSON response")
if not json_schema_valid(obj, schema):
return block("Response missing required fields")
return allow()
```
### Check URLs in Response
```python
def apply_guardrail(inputs, request_data, input_type):
if input_type != "response":
return allow()
for text in inputs["texts"]:
if not all_urls_valid(text):
return block("Response contains invalid URLs")
return allow()
```
### Combine Multiple Checks
```python
def apply_guardrail(inputs, request_data, input_type):
modified = []
for text in inputs["texts"]:
# Redact SSN
text = regex_replace(text, r"\d{3}-\d{2}-\d{4}", "[SSN]")
# Redact credit cards
text = regex_replace(text, r"\d{16}", "[CARD]")
modified.append(text)
# Block SQL in requests
if input_type == "request":
for text in inputs["texts"]:
if contains_code_language(text, ["sql"]):
return block("SQL injection blocked")
return modify(texts=modified)
```
## Sandbox Restrictions
Custom code runs in a restricted environment:
- ❌ No `import` statements
- ❌ No file I/O
- ❌ No network access
- ❌ No `exec()` or `eval()`
- ✅ Only LiteLLM-provided primitives available
## Per-Request Usage
Enable guardrail per request:
```bash
curl -X POST http://localhost:4000/chat/completions \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello"}],
"guardrails": ["block-ssn"]
}'
```
## Default On
Run guardrail on all requests:
```yaml
litellm_settings:
guardrails:
- guardrail_name: block-ssn
litellm_params:
guardrail: custom_code
mode: pre_call
default_on: true
custom_code: |
def apply_guardrail(inputs, request_data, input_type):
...
```

197
docs/my-website/docs/proxy/guardrails/grayswan.md
View file

@ -13,20 +13,26 @@ Cygnal returns a `violation` score between `0` and `1` (higher means more likely
### 1. Obtain Credentials
1. Create a Gray Swan account and generate a Cygnal API key.
1. Log in to our Gray Swan platform and generate a Cygnal API key.
For existing customers, you should already have access to our [platform](https://platform.grayswan.ai).
For new users, please register at this [page](https://hubs.ly/Q03-sX1J0) and we are more than happy to give you an onboarding!
2. Configure environment variables for the LiteLLM proxy host:
```bash
export GRAYSWAN_API_KEY="your-grayswan-key"
export GRAYSWAN_API_BASE="https://api.grayswan.ai"
```
```bash
export GRAYSWAN_API_KEY="your-grayswan-key"
export GRAYSWAN_API_BASE="https://api.grayswan.ai"
```
### 2. Configure `config.yaml`
Add a guardrail entry that references the Gray Swan integration. Below is a balanced example that monitors both input and output but only blocks once the violation score reaches the configured threshold.
Add a guardrail entry that references the Gray Swan integration. Below is our recommmended settings.
```yaml
model_list:
model_list: # this part is a standard litellm configuration for reference
- model_name: openai/gpt-4.1-mini
litellm_params:
model: openai/gpt-4.1-mini
@ -40,13 +46,14 @@ guardrails:
api_key: os.environ/GRAYSWAN_API_KEY
api_base: os.environ/GRAYSWAN_API_BASE # optional
optional_params:
on_flagged_action: monitor # or "block"
on_flagged_action: passthrough # or "block" or "monitor"
violation_threshold: 0.5 # score >= threshold is flagged
reasoning_mode: hybrid # off | hybrid | thinking
categories:
safety: "Detect jailbreaks and policy violations"
policy_id: "your-cygnal-policy-id"
policy_id: "your-cygnal-policy-id" # Optional: Your Cygnal policy ID. Defaults to a content safety policy if empty.
streaming_end_of_stream_only: true # For streaming API, only send the assembled message to Cygnal (post_call only). Defaults to false.
default_on: true
guardrail_timeout: 30 # Defaults to 30 seconds. Change accordingly.
fail_open: true # Defaults to true; set to false to propagate guardrail errors.
general_settings:
master_key: "your-litellm-master-key"
@ -65,13 +72,13 @@ litellm --config config.yaml --port 4000
## Choosing Guardrail Modes
Gray Swan can run during `pre_call`, `during_call`, and `post_call` stages. Combine modes based on your latency and coverage requirements.
Gray Swan can run during `pre_call`, `during_call`, and `post_call` stages. Combine modes based on your latency and coverage requirements.
| Mode | When it Runs | Protects | Typical Use Case |
|--------------|-------------------|-----------------------|------------------|
| `pre_call` | Before LLM call | User input only | Block prompt injection before it reaches the model |
| `during_call`| Parallel to call | User input only | Low-latency monitoring without blocking |
| `post_call` | After response | Full conversation | Scan output for policy violations, leaked secrets, or IPI |
| `post_call` | After response | Model Outputs | Scan output for policy violations, leaked secrets, or IPI |
When using `during_call` with `on_flagged_action: block` or `on_flagged_action: passthrough`:
@ -81,87 +88,110 @@ When using `during_call` with `on_flagged_action: block` or `on_flagged_action:
- The guardrail exception prevents the response from reaching the user, but **does not cancel the running LLM task**
- This means you pay full LLM costs while returning an error/passthrough message to the user
**Recommendation:** For cost-sensitive applications, use `pre_call` and `post_call` instead of `during_call` for blocking or passthrough modes. Reserve `during_call` for `monitor` mode where you want low-latency logging without impacting the user experience.
**Recommendation:** Use `pre_call` and `post_call` instead of `during_call` for `passthrough` (or `block`) `on_flagged_action` (see our recommended configuration above). Reserve `during_call` for `monitor` mode ONLY when you want low-latency logging without impacting the user experience.
<Tabs>
<TabItem value="monitor" label="Monitor Only">
---
```yaml
guardrails:
- guardrail_name: "cygnal-monitor-only"
litellm_params:
guardrail: grayswan
mode: "during_call"
api_key: os.environ/GRAYSWAN_API_KEY
optional_params:
on_flagged_action: monitor
violation_threshold: 0.6
default_on: true
## Work with Claude Code
Follow the official litellm [guide](https://docs.litellm.ai/docs/tutorials/claude_responses_api) on setting up Claude Code with litellm, with the guardrail part mentioned above added to your litellm configuration. Cygnal natively supports coding agent policies defense. Define your own policy or use the provided coding policies on the platform. The example config we show above is also the recommended setup for Claude Code (with the `policy_id` replaced with an appropriate one).
---
## Per-request overrides via `extra_body`
You can override parts of the Gray Swan guardrail configuration on a per-request basis by passing `litellm_metadata.guardrails[*].grayswan.extra_body`.
`extra_body` is merged into the Cygnal request body and takes precedence over specific fields from `config.yaml`, which are `policy_id`, `violation_threshold`, and `reasoning_mode`.
If you include a `metadata` field inside `extra_body`, it is forwarded to the Cygnal API as-is under the request body's `metadata` field.
Example:
```bash
curl -X POST "http://0.0.0.0:4000/v1/messages?beta=true" \
-H "Authorization: Bearer token" \
-H "Content-Type: application/json" \
-d '{
"model": "openrouter/anthropic/claude-sonnet-4.5",
"messages": [{"role": "user", "content": "hello"}],
"litellm_metadata": {
"guardrails": [
{
"cygnal-monitor": {
"extra_body": {
"policy_id": "specific policy id you want to use",
"metadata": {
"user": "health-check"
}
}
}
}
]
}
}'
```
Best for visibility without blocking. Alerts are logged via LiteLLM’s standard logging callbacks.
OpenAI client:
</TabItem>
<TabItem value="block-input" label="Block Input">
```python
from openai import OpenAI
```yaml
guardrails:
- guardrail_name: "cygnal-block-input"
litellm_params:
guardrail: grayswan
mode: "pre_call"
api_key: os.environ/GRAYSWAN_API_KEY
optional_params:
on_flagged_action: block
violation_threshold: 0.4
categories:
pii: "Detect sensitive data"
default_on: true
client = OpenAI(api_key="anything", base_url="http://0.0.0.0:4000")
resp = client.responses.create(
model="openrouter/anthropic/claude-sonnet-4.5",
input="hello",
extra_body={
"litellm_metadata": {
"guardrails": [
{
"cygnal-monitor": {
"extra_body": {
"policy_id": "69038214e5cdb6befc5e991e",
"metadata": {"trace_id": "trace-123"},
}
}
}
]
}
},
)
```
Stops malicious or sensitive prompts before any tokens are generated.
Anthropic client:
</TabItem>
<TabItem value="full-coverage" label="Full Coverage">
```python
from anthropic import Anthropic
```yaml
guardrails:
- guardrail_name: "cygnal-full-coverage"
litellm_params:
guardrail: grayswan
mode: [pre_call, post_call]
api_key: os.environ/GRAYSWAN_API_KEY
optional_params:
on_flagged_action: block
violation_threshold: 0.5
reasoning_mode: thinking
policy_id: "policy-id-from-grayswan"
default_on: true
client = Anthropic(api_key="anything", base_url="http://0.0.0.0:4000")
resp = client.messages.create(
model="openrouter/anthropic/claude-sonnet-4.5",
max_tokens=256,
messages=[{"role": "user", "content": "hello"}],
extra_body={
"litellm_metadata": {
"guardrails": [
{
"cygnal-monitor": {
"extra_body": {
"policy_id": "69038214e5cdb6befc5e991e",
"metadata": {"trace_id": "trace-123"},
}
}
}
]
}
},
)
```
Provides the strongest enforcement by inspecting both prompts and responses.
Notes:
</TabItem>
<TabItem value="passthrough" label="Passthrough Mode">
```yaml
guardrails:
- guardrail_name: "cygnal-passthrough"
litellm_params:
guardrail: grayswan
mode: [pre_call, post_call]
api_key: os.environ/GRAYSWAN_API_KEY
optional_params:
on_flagged_action: passthrough
violation_threshold: 0.5
default_on: true
```
Allows requests to proceed without raising a 400 error when content is flagged. Instead of blocking, the model response content is replaced with a detailed violation message including violation score, violated rules, and detection flags (mutation, IPI). **Supported Response Formats:** OpenAI chat/text completions, Anthropic Messages API. Other response types (embeddings, images, etc.) will log a warning and return unchanged.
</TabItem>
</Tabs>
- The guardrail name (for example, `cygnal-monitor`) must match the `guardrail_name` in `config.yaml`.
- Per-request guardrail overrides may require a premium license, depending on your proxy settings.
---
@ -170,9 +200,14 @@ Allows requests to proceed without raising a 400 error when content is flagged.
| Parameter | Type | Description |
|---------------------------------------|-----------------|-------------|
| `api_key` | string | Gray Swan Cygnal API key. Reads from `GRAYSWAN_API_KEY` if omitted. |
| `api_base` | string | Override for the Gray Swan API base URL. Defaults to `https://api.grayswan.ai` or `GRAYSWAN_API_BASE`. |
| `mode` | string or list | Guardrail stages (`pre_call`, `during_call`, `post_call`). |
| `optional_params.on_flagged_action` | string | `monitor` (log only), `block` (raise `HTTPException`), or `passthrough` (replace response content with violation message, no 400 error). |
| `.optional_params.violation_threshold`| number (0-1) | Scores at or above this value are considered violations. |
| `optional_params.violation_threshold` | number (0-1) | Scores at or above this value are considered violations. |
| `optional_params.reasoning_mode` | string | `off`, `hybrid`, or `thinking`. Enables Cygnal's reasoning capabilities. |
| `optional_params.categories` | object | Map of custom category names to descriptions. |
| `optional_params.policy_id` | string | Gray Swan policy identifier. |
| `guardrail_timeout` | number | Timeout in seconds for the Cygnal request. Defaults to 30. |
| `fail_open` | boolean | If true, errors contacting Cygnal are logged and the request proceeds; if false, errors propagate. Defaults to treu. |
| `streaming_end_of_stream_only` | boolean | For streaming `post_call`, only send the final assembled response to Cygnal. Defaults to false. |
| `default_on` | boolean | Run the guardrail on every request by default. |

18
docs/my-website/docs/realtime.md
View file

@ -3,11 +3,12 @@ import TabItem from '@theme/TabItem';
# /realtime
Use this to loadbalance across Azure + OpenAI.
Use this to loadbalance across Azure + OpenAI + xAI and more.
Supported Providers:
- OpenAI
- Azure
- xAI ([see full docs](/docs/providers/xai_realtime))
- Google AI Studio (Gemini)
- Vertex AI
- Bedrock
@ -46,6 +47,21 @@ model_list:
api_key: os.environ/OPENAI_API_KEY
```
</TabItem>
<TabItem value="xai" label="xAI Grok Voice Agent">
```yaml
model_list:
- model_name: grok-voice-agent
litellm_params:
model: xai/grok-4-1-fast-non-reasoning
api_key: os.environ/XAI_API_KEY
model_info:
mode: realtime
```
**[See full xAI Realtime documentation →](/docs/providers/xai_realtime)**
</TabItem>
</Tabs>

99
docs/my-website/docs/tutorials/copilotkit_sdk.md Normal file
View file

@ -0,0 +1,99 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# CopilotKit SDK with LiteLLM
Use CopilotKit SDK with any LLM provider through LiteLLM Proxy.
> **Note:** CopilotKit SDK integration with LiteLLM Proxy works with LiteLLM v1.81.7-nightly or higher.
## Quick Start
### 1. Add Model to Config
```yaml title="config.yaml"
model_list:
- model_name: claude-sonnet-4-5
litellm_params:
model: "anthropic/claude-sonnet-4-5-20250514-v1:0"
api_key: "os.environ/ANTHROPIC_API_KEY"
```
### 2. Start LiteLLM Proxy
```bash
litellm --config config.yaml
```
### 3. Use CopilotKit SDK
```typescript
import OpenAI from "openai";
import {
CopilotRuntime,
OpenAIAdapter,
copilotRuntimeNextJSAppRouterEndpoint,
} from "@copilotkit/runtime";
import { NextRequest } from "next/server";
const model = "claude-sonnet-4-5";
const openai = new OpenAI({
apiKey: process.env.OPENAI_API_KEY || "sk-12345",
baseURL: process.env.OPENAI_BASE_URL || "http://localhost:4000/v1",
});
const serviceAdapter = new OpenAIAdapter({ openai, model });
const runtime = new CopilotRuntime();
export const POST = async (req: NextRequest) => {
const { handleRequest } = copilotRuntimeNextJSAppRouterEndpoint({
runtime,
serviceAdapter,
endpoint: "/api/copilotkit",
});
return handleRequest(req);
};
```
### 4. Test
```bash
curl -X POST http://localhost:3000/api/copilotkit \
-H "Content-Type: application/json" \
-d '{
"method": "agent/run",
"params": {
"agentId": "default"
},
"runId": "your_run_id",
"threadId": "your_thread_id",
"runId": ""your_run_id"",
"tools": [],
"context": [],
"forwardedProps": {},
"state": {},
"messages": [
{
"id": "166e573e-f7c6-4c0f-8685-04dbefec18be",
"content": "Hi",
"role": "user"
}
]
}
}'
```
## Environment Variables
| Variable | Value | Description |
|----------|-------|-------------|
| `OPENAI_API_KEY` | `sk-12345` | Your LiteLLM API key |
| `OPENAI_BASE_URL` | `http://localhost:4000/v1` | LiteLLM proxy URL |
## Related Resources
- [CopilotKit Documentation](https://docs.copilotkit.ai)
- [LiteLLM Proxy Quick Start](../proxy/quick_start)

190
docs/my-website/docs/tutorials/livekit_xai_realtime.md Normal file
View file

@ -0,0 +1,190 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# LiveKit xAI Realtime Voice Agent
Use LiveKit's xAI Grok Voice Agent plugin with LiteLLM Proxy to build low-latency voice AI agents.
The LiveKit Agents framework provides tools for building real-time voice and video AI applications. By routing through LiteLLM Proxy, you get unified access to multiple realtime voice providers, cost tracking, rate limiting, and more.
## Quick Start
### 1. Install Dependencies
```bash
pip install livekit-agents[xai]
```
### 2. Start LiteLLM Proxy
Create a config file with your xAI realtime model:
```yaml title="config.yaml" showLineNumbers
model_list:
- model_name: grok-voice-agent
litellm_params:
model: xai/grok-2-vision-1212
api_key: os.environ/XAI_API_KEY
model_info:
mode: realtime
litellm_settings:
drop_params: True
general_settings:
master_key: sk-1234 # Change this to a secure key
```
Start the proxy:
```bash
litellm --config config.yaml --port 4000
```
### 3. Configure LiveKit xAI Plugin
Point LiveKit's xAI plugin to your LiteLLM proxy:
```python
from livekit.plugins import xai
# Configure xAI to use LiteLLM proxy
model = xai.realtime.RealtimeModel(
voice="ara", # Voice option
api_key="sk-1234", # Your LiteLLM proxy master key
base_url="http://localhost:4000", # LiteLLM proxy URL
)
```
## Complete Example
Here's a complete working example:
<Tabs>
<TabItem value="python" label="Python Client">
```python
#!/usr/bin/env python3
"""
Simple xAI realtime voice agent through LiteLLM proxy.
"""
import asyncio
import json
import websockets
PROXY_URL = "ws://localhost:4000/v1/realtime"
API_KEY = "sk-1234"
MODEL = "grok-voice-agent"
async def run_voice_agent():
"""Connect to xAI realtime API through LiteLLM proxy"""
url = f"{PROXY_URL}?model={MODEL}"
headers = {"Authorization": f"Bearer {API_KEY}"}
async with websockets.connect(url, extra_headers=headers) as ws:
# Wait for initial connection event
initial = json.loads(await ws.recv())
print(f"✅ Connected: {initial['type']}")
# Send user message
await ws.send(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{
"type": "input_text",
"text": "Hello! Tell me a joke."
}]
}
}))
# Request response
await ws.send(json.dumps({
"type": "response.create",
"response": {"modalities": ["text", "audio"]}
}))
# Collect response
transcript = []
async for message in ws:
event = json.loads(message)
# Capture text response
if event['type'] == 'response.output_audio_transcript.delta':
transcript.append(event['delta'])
print(event['delta'], end='', flush=True)
# Done when response completes
elif event['type'] == 'response.done':
break
print(f"\n\n✅ Full response: {''.join(transcript)}")
if __name__ == "__main__":
asyncio.run(run_voice_agent())
```
</TabItem>
<TabItem value="livekit" label="LiveKit Agent">
```python
from livekit.agents import Agent, AgentSession, WorkerOptions, cli
from livekit.plugins import xai
class VoiceAgent(Agent):
def __init__(self):
super().__init__(
instructions="You are a helpful voice assistant.",
llm=xai.realtime.RealtimeModel(
voice="ara",
api_key="sk-1234",
base_url="http://localhost:4000",
),
)
if __name__ == "__main__":
cli.run_app(
WorkerOptions(
agent_factory=VoiceAgent,
)
)
```
</TabItem>
</Tabs>
## Running the Example
1. **Start LiteLLM Proxy** (if not already running):
```bash
litellm --config config.yaml --port 4000
```
2. **Run the example**:
```bash
python your_script.py
```
## Expected Output
```
✅ Connected: conversation.created
Hello! Here's a joke for you: Why don't scientists trust atoms?
Because they make up everything!
✅ Full response: Hello! Here's a joke for you: Why don't scientists trust atoms? Because they make up everything!
```
## Complete Working Example
**[LiveKit Agent SDK Cookbook](https://github.com/BerriAI/litellm/tree/main/cookbook/livekit_agent_sdk)**
## Learn More
- [xAI Realtime API](/docs/providers/xai_realtime)
- [LiveKit xAI Plugin](https://docs.livekit.io/agents/models/realtime/plugins/xai/)
- [LiteLLM Realtime API](/docs/realtime)

BIN
docs/my-website/img/okta_access_policies.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 82 KiB

BIN
docs/my-website/img/okta_authorization_server.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 52 KiB

BIN
docs/my-website/img/okta_client_credentials.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 64 KiB

BIN
docs/my-website/img/okta_redirect_uri.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 60 KiB

BIN
docs/my-website/img/okta_security_api.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 38 KiB

13
docs/my-website/sidebars.js
View file

@ -79,6 +79,7 @@ const sidebars = {
"proxy/guardrails/panw_prisma_airs",
"proxy/guardrails/secret_detection",
"proxy/guardrails/custom_guardrail",
"proxy/guardrails/custom_code_guardrail",
"proxy/guardrails/prompt_injection",
"proxy/guardrails/tool_permission",
"proxy/guardrails/zscaler_ai_guard",
@ -150,7 +151,9 @@ const sidebars = {
},
items: [
"tutorials/claude_agent_sdk",
"tutorials/copilotkit_sdk",
"tutorials/google_adk",
"tutorials/livekit_xai_realtime",
]
},
@ -469,6 +472,7 @@ const sidebars = {
label: "/a2a - A2A Agent Gateway",
items: [
"a2a",
"a2a_invoking_agents",
"a2a_cost_tracking",
"a2a_agent_permissions"
],
@ -850,7 +854,14 @@ const sidebars = {
"providers/watsonx/audio_transcription",
]
},
"providers/xai",
{
type: "category",
label: "xAI",
items: [
"providers/xai",
"providers/xai_realtime",
]
},
"providers/xiaomi_mimo",
"providers/xinference",
"providers/zai",

2
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -53,7 +53,7 @@ class CheckBatchCost:
jobs = await self.prisma_client.db.litellm_managedobjecttable.find_many(
where={
"status": "validating",
"status": {"in": ["validating", "in_progress", "finalizing"]},
"file_purpose": "batch",
}
)

97
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -166,7 +166,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"updated_by": user_api_key_dict.user_id,
"status": file_object.status,
},
"update": {}, # don't do anything if it already exists
"update": {
"file_object": file_object.model_dump_json(),
"status": file_object.status,
"updated_by": user_api_key_dict.user_id,
}, # FIX: Update status and file_object on every operation to keep state in sync
},
)
@ -354,6 +358,31 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
return False
async def check_file_ids_access(
self, file_ids: List[str], user_api_key_dict: UserAPIKeyAuth
) -> None:
"""
Check if the user has access to a list of file IDs.
Only checks managed (unified) file IDs.
Args:
file_ids: List of file IDs to check access for
user_api_key_dict: User API key authentication details
Raises:
HTTPException: If user doesn't have access to any of the files
"""
for file_id in file_ids:
is_unified_file_id = _is_base64_encoded_unified_file_id(file_id)
if is_unified_file_id:
if not await self.can_user_call_unified_file_id(
file_id, user_api_key_dict
):
raise HTTPException(
status_code=403,
detail=f"User {user_api_key_dict.user_id} does not have access to the file {file_id}",
)
async def async_pre_call_hook( # noqa: PLR0915
self,
user_api_key_dict: UserAPIKeyAuth,
@ -387,6 +416,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if messages:
file_ids = self.get_file_ids_from_messages(messages)
if file_ids:
# Check user has access to all managed files
await self.check_file_ids_access(file_ids, user_api_key_dict)
# Check if any files are stored in storage backends and need base64 conversion
# This is needed for Vertex AI/Gemini which requires base64 content
is_vertex_ai = model and ("vertex_ai" in model or "gemini" in model.lower())
@ -402,15 +434,27 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
data["model_file_id_mapping"] = model_file_id_mapping
elif call_type == CallTypes.aresponses.value or call_type == CallTypes.responses.value:
# Handle managed files in responses API input
# Handle managed files in responses API input and tools
file_ids = []
# Extract file IDs from input parameter
input_data = data.get("input")
if input_data:
file_ids = self.get_file_ids_from_responses_input(input_data)
if file_ids:
model_file_id_mapping = await self.get_model_file_id_mapping(
file_ids, user_api_key_dict.parent_otel_span
)
data["model_file_id_mapping"] = model_file_id_mapping
file_ids.extend(self.get_file_ids_from_responses_input(input_data))
# Extract file IDs from tools parameter (e.g., code_interpreter container)
tools = data.get("tools")
if tools:
file_ids.extend(self.get_file_ids_from_responses_tools(tools))
if file_ids:
# Check user has access to all managed files
await self.check_file_ids_access(file_ids, user_api_key_dict)
model_file_id_mapping = await self.get_model_file_id_mapping(
file_ids, user_api_key_dict.parent_otel_span
)
data["model_file_id_mapping"] = model_file_id_mapping
elif call_type == CallTypes.afile_content.value:
retrieve_file_id = cast(Optional[str], data.get("file_id"))
potential_file_id = (
@ -460,8 +504,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
if retrieve_object_id
else False
)
print(f"🔥potential_llm_object_id: {potential_llm_object_id}")
print(f"🔥retrieve_object_id: {retrieve_object_id}")
if potential_llm_object_id and retrieve_object_id:
## VALIDATE USER HAS ACCESS TO THE OBJECT ##
if not await self.can_user_call_unified_object_id(
@ -614,6 +656,41 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
return file_ids
def get_file_ids_from_responses_tools(
self, tools: List[Dict[str, Any]]
) -> List[str]:
"""
Gets file ids from responses API tools parameter.
The tools can contain code_interpreter with container.file_ids:
[
{
"type": "code_interpreter",
"container": {"type": "auto", "file_ids": ["file-123", "file-456"]}
}
]
"""
file_ids: List[str] = []
if not isinstance(tools, list):
return file_ids
for tool in tools:
if not isinstance(tool, dict):
continue
# Check for code_interpreter with container file_ids
if tool.get("type") == "code_interpreter":
container = tool.get("container")
if isinstance(container, dict):
container_file_ids = container.get("file_ids")
if isinstance(container_file_ids, list):
for file_id in container_file_ids:
if isinstance(file_id, str):
file_ids.append(file_id)
return file_ids
async def get_model_file_id_mapping(
self, file_ids: List[str], litellm_parent_otel_span: Span
) -> dict:

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.30-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.30.tar.gz vendored Normal file
View file

Binary file not shown.

6
litellm-proxy-extras/litellm_proxy_extras/migrations/20260205091235_allow_team_guardrail_config/migration.sql Normal file
View file

@ -0,0 +1,6 @@
-- AlterTable
ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN "allow_team_guardrail_config" BOOLEAN NOT NULL DEFAULT false;
-- AlterTable
ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN "allow_team_guardrail_config" BOOLEAN NOT NULL DEFAULT false;

5
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -129,6 +129,7 @@ model LiteLLM_TeamTable {
team_member_permissions String[] @default([])
policies String[] @default([])
model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases
allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
litellm_model_table LiteLLM_ModelTable? @relation(fields: [model_id], references: [id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])
@ -160,7 +161,8 @@ model LiteLLM_DeletedTeamTable {
team_member_permissions String[] @default([])
policies String[] @default([])
model_id Int? // id for LiteLLM_ModelTable -> stores team-level model aliases
allow_team_guardrail_config Boolean @default(false)
// Original timestamps from team creation/updates
created_at DateTime? @map("created_at")
updated_at DateTime? @map("updated_at")
@ -774,6 +776,7 @@ model LiteLLM_GuardrailsTable {
guardrail_name String @unique
litellm_params Json
guardrail_info Json?
team_id String?
created_at DateTime @default(now())
updated_at DateTime @updatedAt
}

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.29"
version = "0.4.30"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.29"
version = "0.4.30"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

2
litellm/__init__.py
View file

@ -261,6 +261,8 @@ extra_spend_tag_headers: Optional[List[str]] = None
in_memory_llm_clients_cache: "LLMClientCache"
safe_memory_mode: bool = False
enable_azure_ad_token_refresh: Optional[bool] = False
# Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy
proxy_auth: Optional[Any] = None
### DEFAULT AZURE API VERSION ###
AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest
### DEFAULT WATSONX API VERSION ###

3
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -329,6 +329,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
else:
request_data[key] = value
if headers:
request_data["extra_headers"] = headers
return request_data
@staticmethod

8
litellm/constants.py
View file

@ -81,6 +81,11 @@ MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH = int(
os.getenv("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150)
)
LITELLM_UI_ALLOW_HEADERS = [
"x-litellm-semantic-filter",
"x-litellm-semantic-filter-tools",
]
# Gemini model-specific minimal thinking budget constants
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH = int(
os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH", 1)
@ -99,6 +104,9 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET = int(
os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128)
)
# Provider-specific API base URLs
XAI_API_BASE = "https://api.x.ai/v1"
DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = int(
os.getenv("DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET", 1024)
)

9
litellm/integrations/custom_guardrail.py
View file

@ -475,11 +475,18 @@ class CustomGuardrail(CustomLogger):
guardrail_config: DynamicGuardrailParams = DynamicGuardrailParams(
**guardrail[self.guardrail_name]
)
extra_body = guardrail_config.get("extra_body", {})
if self._validate_premium_user() is not True:
if isinstance(extra_body, dict) and extra_body:
verbose_logger.warning(
"Guardrail %s: ignoring dynamic extra_body keys %s because premium_user is False",
self.guardrail_name,
list(extra_body.keys()),
)
return {}
# Return the extra_body if it exists, otherwise empty dict
return guardrail_config.get("extra_body", {})
return extra_body
return {}

79
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -8,9 +8,8 @@ from litellm.integrations.arize import _utils
from litellm.integrations.langfuse.langfuse_otel_attributes import (
LangfuseLLMObsOTELAttributes,
)
from litellm.integrations.opentelemetry import OpenTelemetry
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.types.integrations.langfuse_otel import (
LangfuseOtelConfig,
LangfuseSpanAttributes,
)
from litellm.types.utils import StandardCallbackDynamicParams
@ -18,17 +17,8 @@ from litellm.types.utils import StandardCallbackDynamicParams
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.integrations.opentelemetry import (
OpenTelemetryConfig as _OpenTelemetryConfig,
)
from litellm.types.integrations.arize import Protocol as _Protocol
Protocol = _Protocol
OpenTelemetryConfig = _OpenTelemetryConfig
Span = Union[_Span, Any]
else:
Protocol = Any
OpenTelemetryConfig = Any
Span = Any
@ -37,8 +27,12 @@ LANGFUSE_CLOUD_US_ENDPOINT = "https://us.cloud.langfuse.com/api/public/otel"
class LangfuseOtelLogger(OpenTelemetry):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def __init__(self, config=None, *args, **kwargs):
# Prevent LangfuseOtelLogger from modifying global environment variables by constructing config manually
# and passing it to the parent OpenTelemetry class
if config is None:
config = self._create_open_telemetry_config_from_langfuse_env()
super().__init__(config=config, *args, **kwargs)
@staticmethod
def set_langfuse_otel_attributes(span: Span, kwargs, response_obj):
@ -114,6 +108,10 @@ class LangfuseOtelLogger(OpenTelemetry):
for key, enum_attr in mapping.items():
if key in metadata and metadata[key] is not None:
value = metadata[key]
if key == "trace_id" and isinstance(value, str):
# trace_id must be 32 hex char no dashes for langfuse : Litellm sends uuid with dashes (might be breaking at some point)
value = value.replace("-", "")
if isinstance(value, (list, dict)):
try:
value = json.dumps(value)
@ -265,8 +263,47 @@ class LangfuseOtelLogger(OpenTelemetry):
"""
return os.environ.get("LANGFUSE_OTEL_HOST") or os.environ.get("LANGFUSE_HOST")
def _create_open_telemetry_config_from_langfuse_env(self) -> OpenTelemetryConfig:
"""
Creates OpenTelemetryConfig from Langfuse environment variables.
Does NOT modify global environment variables.
"""
from litellm.integrations.opentelemetry import OpenTelemetryConfig
public_key = os.environ.get("LANGFUSE_PUBLIC_KEY", None)
secret_key = os.environ.get("LANGFUSE_SECRET_KEY", None)
if not public_key or not secret_key:
# If no keys, return default from env (likely logging to console or something else)
return OpenTelemetryConfig.from_env()
# Determine endpoint - default to US cloud
langfuse_host = LangfuseOtelLogger._get_langfuse_otel_host()
if langfuse_host:
# If LANGFUSE_HOST is provided, construct OTEL endpoint from it
if not langfuse_host.startswith("http"):
langfuse_host = "https://" + langfuse_host
endpoint = f"{langfuse_host.rstrip('/')}/api/public/otel"
verbose_logger.debug(f"Using Langfuse OTEL endpoint from host: {endpoint}")
else:
# Default to US cloud endpoint
endpoint = LANGFUSE_CLOUD_US_ENDPOINT
verbose_logger.debug(f"Using Langfuse US cloud endpoint: {endpoint}")
auth_header = LangfuseOtelLogger._get_langfuse_authorization_header(
public_key=public_key, secret_key=secret_key
)
otlp_auth_headers = f"Authorization={auth_header}"
return OpenTelemetryConfig(
exporter="otlp_http",
endpoint=endpoint,
headers=otlp_auth_headers,
)
@staticmethod
def get_langfuse_otel_config() -> LangfuseOtelConfig:
def get_langfuse_otel_config() -> "OpenTelemetryConfig":
"""
Retrieves the Langfuse OpenTelemetry configuration based on environment variables.
@ -276,7 +313,7 @@ class LangfuseOtelLogger(OpenTelemetry):
LANGFUSE_HOST: Optional. Custom Langfuse host URL. Defaults to US cloud.
Returns:
LangfuseOtelConfig: A Pydantic model containing Langfuse OTEL configuration.
OpenTelemetryConfig: A Pydantic model containing Langfuse OTEL configuration.
Raises:
ValueError: If required keys are missing.
@ -308,12 +345,14 @@ class LangfuseOtelLogger(OpenTelemetry):
)
otlp_auth_headers = f"Authorization={auth_header}"
# Set standard OTEL environment variables
os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint
os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = otlp_auth_headers
# Prevent modification of global env vars which causes leakage
# os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint
# os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = otlp_auth_headers
return LangfuseOtelConfig(
otlp_auth_headers=otlp_auth_headers, protocol="otlp_http"
return OpenTelemetryConfig(
exporter="otlp_http",
endpoint=endpoint,
headers=otlp_auth_headers,
)
@staticmethod

30
litellm/integrations/opentelemetry.py
View file

@ -599,9 +599,9 @@ class OpenTelemetry(CustomLogger):
def _get_dynamic_otel_headers_from_kwargs(self, kwargs) -> Optional[dict]:
"""Extract dynamic headers from kwargs if available."""
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params")
)
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params")
if not standard_callback_dynamic_params:
return None
@ -619,7 +619,9 @@ class OpenTelemetry(CustomLogger):
# Prevents thread exhaustion by reusing providers for the same credential sets (e.g. per-team keys)
cache_key = str(sorted(dynamic_headers.items()))
if cache_key in self._tracer_provider_cache:
return self._tracer_provider_cache[cache_key].get_tracer(LITELLM_TRACER_NAME)
return self._tracer_provider_cache[cache_key].get_tracer(
LITELLM_TRACER_NAME
)
# Create a temporary tracer provider with dynamic headers
temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config))
@ -674,7 +676,10 @@ class OpenTelemetry(CustomLogger):
kwargs, response_obj, start_time, end_time, span
)
# Ensure proxy-request parent span is annotated with the actual operation kind
if parent_span is not None and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME:
if (
parent_span is not None
and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
):
self.set_attributes(parent_span, kwargs, response_obj)
else:
# Do not create primary span (keep hierarchy shallow when parent exists)
@ -1003,14 +1008,11 @@ class OpenTelemetry(CustomLogger):
# TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
from opentelemetry._logs import SeverityNumber, get_logger, get_logger_provider
try:
from opentelemetry.sdk._logs import (
LogRecord as SdkLogRecord, # type: ignore[attr-defined] # OTEL < 1.39.0
)
from opentelemetry.sdk._logs import LogRecord as SdkLogRecord # type: ignore[attr-defined] # OTEL < 1.39.0
except ImportError:
from opentelemetry.sdk._logs._internal import (
LogRecord as SdkLogRecord, # OTEL >= 1.39.0
)
from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord # type: ignore[attr-defined, no-redef] # OTEL >= 1.39.0
otel_logger = get_logger(LITELLM_LOGGER_NAME)
@ -1618,7 +1620,6 @@ class OpenTelemetry(CustomLogger):
for idx, choice in enumerate(response_obj.get("choices")):
if choice.get("finish_reason"):
message = choice.get("message")
tool_calls = message.get("tool_calls")
if tool_calls:
@ -1631,7 +1632,9 @@ class OpenTelemetry(CustomLogger):
)
except Exception as e:
self.handle_callback_failure(callback_name=self.callback_name or "opentelemetry")
self.handle_callback_failure(
callback_name=self.callback_name or "opentelemetry"
)
verbose_logger.exception(
"OpenTelemetry logging error in set_attributes %s", str(e)
)
@ -1722,6 +1725,7 @@ class OpenTelemetry(CustomLogger):
def set_raw_request_attributes(self, span: Span, kwargs, response_obj):
try:
self.set_attributes(span, kwargs, response_obj)
kwargs.get("optional_params", {})
litellm_params = kwargs.get("litellm_params", {}) or {}
custom_llm_provider = litellm_params.get("custom_llm_provider", "Unknown")

169
litellm/integrations/prometheus.py
View file

@ -1683,6 +1683,108 @@ class PrometheusLogger(CustomLogger):
)
pass
def _safe_get(self, obj: Any, key: str, default: Any = None) -> Any:
"""Get value from dict or Pydantic model."""
if obj is None:
return default
if isinstance(obj, dict):
return obj.get(key, default)
return getattr(obj, key, default)
def _extract_deployment_failure_label_values(
self, request_kwargs: dict
) -> Dict[str, Optional[str]]:
"""
Extract label values for deployment failure metrics from all available
sources in request_kwargs. Falls back to litellm_params metadata and
user_api_key_auth when standard_logging_payload has None values.
"""
standard_logging_payload = (
request_kwargs.get("standard_logging_object", {}) or {}
)
_litellm_params = request_kwargs.get("litellm_params", {}) or {}
_metadata_raw = self._safe_get(standard_logging_payload, "metadata") or {}
if isinstance(_metadata_raw, dict):
_metadata = _metadata_raw
else:
_metadata = {
"user_api_key_alias": getattr(
_metadata_raw, "user_api_key_alias", None
),
"user_api_key_team_id": getattr(
_metadata_raw, "user_api_key_team_id", None
),
"user_api_key_team_alias": getattr(
_metadata_raw, "user_api_key_team_alias", None
),
"user_api_key_hash": getattr(_metadata_raw, "user_api_key_hash", None),
"requester_ip_address": getattr(
_metadata_raw, "requester_ip_address", None
),
"user_agent": getattr(_metadata_raw, "user_agent", None),
}
_litellm_params_metadata = _litellm_params.get("metadata", {}) or {}
# Extract user_api_key_auth if present (proxy injects this, skipped in merge)
user_api_key_auth = _litellm_params_metadata.get("user_api_key_auth")
def _get_api_key_alias() -> Optional[str]:
val = _metadata.get("user_api_key_alias")
if val is not None:
return val
val = _litellm_params_metadata.get("user_api_key_alias")
if val is not None:
return val
if user_api_key_auth is not None:
return getattr(user_api_key_auth, "key_alias", None)
return None
def _get_team_id() -> Optional[str]:
val = _metadata.get("user_api_key_team_id")
if val is not None:
return val
val = _litellm_params_metadata.get("user_api_key_team_id")
if val is not None:
return val
if user_api_key_auth is not None:
return getattr(user_api_key_auth, "team_id", None)
return None
def _get_team_alias() -> Optional[str]:
val = _metadata.get("user_api_key_team_alias")
if val is not None:
return val
val = _litellm_params_metadata.get("user_api_key_team_alias")
if val is not None:
return val
if user_api_key_auth is not None:
return getattr(user_api_key_auth, "team_alias", None)
return None
def _get_hashed_api_key() -> Optional[str]:
val = _metadata.get("user_api_key_hash")
if val is not None:
return val
val = _litellm_params_metadata.get("user_api_key_hash")
if val is not None:
return val
if user_api_key_auth is not None:
return getattr(user_api_key_auth, "api_key", None) or getattr(
user_api_key_auth, "api_key_hash", None
)
return None
return {
"api_key_alias": _get_api_key_alias(),
"team": _get_team_id(),
"team_alias": _get_team_alias(),
"hashed_api_key": _get_hashed_api_key(),
"client_ip": _metadata.get("requester_ip_address")
or _litellm_params_metadata.get("requester_ip_address"),
"user_agent": _metadata.get("user_agent")
or _litellm_params_metadata.get("user_agent"),
}
def set_llm_deployment_failure_metrics(self, request_kwargs: dict):
"""
Sets Failure metrics when an LLM API call fails
@ -1707,6 +1809,21 @@ class PrometheusLogger(CustomLogger):
model_id = standard_logging_payload.get("model_id", None)
exception = request_kwargs.get("exception", None)
# Fallback: model_id from litellm_metadata.model_info
if model_id is None:
_model_info = (
(_litellm_params.get("litellm_metadata") or {}).get("model_info")
or (_litellm_params.get("metadata") or {}).get("model_info")
or {}
)
model_id = _model_info.get("id")
# Fallback: model_group from litellm_metadata
if model_group is None:
model_group = (_litellm_params.get("litellm_metadata") or {}).get(
"model_group"
) or (_litellm_params.get("metadata") or {}).get("model_group")
llm_provider = _litellm_params.get("custom_llm_provider", None)
if self._should_skip_metrics_for_invalid_key(
@ -1714,9 +1831,37 @@ class PrometheusLogger(CustomLogger):
standard_logging_payload=standard_logging_payload,
):
return
hashed_api_key = standard_logging_payload.get("metadata", {}).get(
# Extract context labels from all available sources (fix for None labels)
fallback_values = self._extract_deployment_failure_label_values(
request_kwargs
)
_metadata = standard_logging_payload.get("metadata", {}) or {}
hashed_api_key = fallback_values.get("hashed_api_key") or _metadata.get(
"user_api_key_hash"
)
api_key_alias = fallback_values.get("api_key_alias") or _metadata.get(
"user_api_key_alias"
)
team = fallback_values.get("team") or _metadata.get("user_api_key_team_id")
team_alias = fallback_values.get("team_alias") or _metadata.get(
"user_api_key_team_alias"
)
client_ip = fallback_values.get("client_ip") or _metadata.get(
"requester_ip_address"
)
user_agent = fallback_values.get("user_agent") or _metadata.get(
"user_agent"
)
# exception_status: prefer status_code, fallback to exception class for known types
exception_status = None
if exception is not None:
exception_status = str(getattr(exception, "status_code", None))
if exception_status == "None" or not exception_status:
code = getattr(exception, "code", None)
if code is not None:
exception_status = str(code)
# Create enum_values for the label factory (always create for use in different metrics)
enum_values = UserAPIKeyLabelValues(
@ -1724,26 +1869,18 @@ class PrometheusLogger(CustomLogger):
model_id=model_id,
api_base=api_base,
api_provider=llm_provider,
exception_status=(
str(getattr(exception, "status_code", None)) if exception else None
),
exception_status=exception_status,
exception_class=(
self._get_exception_class_name(exception) if exception else None
),
requested_model=model_group,
requested_model=model_group or litellm_model_name,
hashed_api_key=hashed_api_key,
api_key_alias=standard_logging_payload["metadata"][
"user_api_key_alias"
],
team=standard_logging_payload["metadata"]["user_api_key_team_id"],
team_alias=standard_logging_payload["metadata"][
"user_api_key_team_alias"
],
api_key_alias=api_key_alias,
team=team,
team_alias=team_alias,
tags=standard_logging_payload.get("request_tags", []),
client_ip=standard_logging_payload["metadata"].get(
"requester_ip_address"
),
user_agent=standard_logging_payload["metadata"].get("user_agent"),
client_ip=client_ip,
user_agent=user_agent,
)
"""

54
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -1,8 +1,35 @@
from typing import Dict, Optional
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import StandardCallbackDynamicParams
# Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
_supported_callback_params = [
"langfuse_public_key",
"langfuse_secret",
"langfuse_secret_key",
"langfuse_host",
"langfuse_prompt_version",
"gcs_bucket_name",
"gcs_path_service_account",
"langsmith_api_key",
"langsmith_project",
"langsmith_base_url",
"langsmith_sampling_rate",
"langsmith_tenant_id",
"humanloop_api_key",
"arize_api_key",
"arize_space_key",
"arize_space_id",
"posthog_api_key",
"posthog_host",
"braintrust_api_key",
"braintrust_project",
"braintrust_host",
"slack_webhook_url",
"lunary_public_key",
"turn_off_message_logging",
]
def initialize_standard_callback_dynamic_params(
kwargs: Optional[Dict] = None,
@ -15,13 +42,10 @@ def initialize_standard_callback_dynamic_params(
standard_callback_dynamic_params = StandardCallbackDynamicParams()
if kwargs:
_supported_callback_params = (
StandardCallbackDynamicParams.__annotations__.keys()
)
# 1. Check top-level kwargs
for param in _supported_callback_params:
if param in kwargs:
_param_value = kwargs.pop(param)
_param_value = kwargs.get(param)
if (
_param_value is not None
and isinstance(_param_value, str)
@ -30,4 +54,22 @@ def initialize_standard_callback_dynamic_params(
_param_value = get_secret_str(secret_name=_param_value)
standard_callback_dynamic_params[param] = _param_value # type: ignore
# 2. Fallback: check "metadata" or "litellm_params" -> "metadata"
metadata = (kwargs.get("metadata") or {}).copy()
litellm_params = kwargs.get("litellm_params") or {}
if isinstance(litellm_params, dict):
metadata.update(litellm_params.get("metadata") or {})
if isinstance(metadata, dict):
for param in _supported_callback_params:
if param not in standard_callback_dynamic_params and param in metadata:
_param_value = metadata.get(param)
if (
_param_value is not None
and isinstance(_param_value, str)
and "os.environ/" in _param_value
):
_param_value = get_secret_str(secret_name=_param_value)
standard_callback_dynamic_params[param] = _param_value # type: ignore
return standard_callback_dynamic_params

16
litellm/litellm_core_utils/litellm_logging.py
View file

@ -3917,18 +3917,6 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
return langfuse_logger # type: ignore
elif logging_integration == "langfuse_otel":
from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
from litellm.integrations.opentelemetry import (
OpenTelemetry,
OpenTelemetryConfig,
)
langfuse_otel_config = LangfuseOtelLogger.get_langfuse_otel_config()
# The endpoint and headers are now set as environment variables by get_langfuse_otel_config()
otel_config = OpenTelemetryConfig(
exporter=langfuse_otel_config.protocol,
headers=langfuse_otel_config.otlp_auth_headers,
)
for callback in _in_memory_loggers:
if (
@ -3936,8 +3924,10 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
and callback.callback_name == "langfuse_otel"
):
return callback # type: ignore
# Allow LangfuseOtelLogger to initialize its own config safely
# This prevents startup crashes if LANGFUSE keys are not in env (e.g. for dynamic usage)
_otel_logger = LangfuseOtelLogger(
config=otel_config, callback_name="langfuse_otel"
config=None, callback_name="langfuse_otel"
)
_in_memory_loggers.append(_otel_logger)
return _otel_logger # type: ignore

21
litellm/litellm_core_utils/logging_callback_manager.py
View file

@ -114,6 +114,27 @@ class LoggingCallbackManager:
for c in remove_list:
callback_list.remove(c)
def remove_callbacks_by_type(self, callback_list, callback_type):
"""
Remove all callbacks of a specific type from a callback list.
Args:
callback_list: The list to remove callbacks from (e.g., litellm.callbacks)
callback_type: The class type to match (e.g., SemanticToolFilterHook)
Example:
litellm.logging_callback_manager.remove_callbacks_by_type(
litellm.callbacks, SemanticToolFilterHook
)
"""
if not isinstance(callback_list, list):
return
remove_list = [c for c in callback_list if isinstance(c, callback_type)]
for c in remove_list:
callback_list.remove(c)
def _add_string_callback_to_list(
self, callback: str, parent_list: List[Union[CustomLogger, Callable, str]]
):

111
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -443,13 +443,21 @@ def update_messages_with_model_file_ids(
def update_responses_input_with_model_file_ids(
input: Any,
model_id: Optional[str] = None,
model_file_id_mapping: Optional[Dict[str, Dict[str, str]]] = None,
) -> Union[str, List[Dict[str, Any]]]:
"""
Updates responses API input with provider-specific file IDs.
File IDs are always inside the content array, not as direct input_file items.
For managed files (unified file IDs), decodes the base64-encoded unified file ID
and extracts the llm_output_file_id directly.
For managed files (unified file IDs), uses model_file_id_mapping if provided,
otherwise decodes the base64-encoded unified file ID and extracts the llm_output_file_id directly.
Args:
input: The responses API input parameter
model_id: The model ID to use for looking up provider-specific file IDs
model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs
Format: {"litellm_file_id": {"model_id": "provider_file_id"}}
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
@ -479,22 +487,35 @@ def update_responses_input_with_model_file_ids(
):
file_id = content_item.get("file_id")
if file_id:
# Check if this is a managed file ID (base64-encoded unified file ID)
is_unified_file_id = _is_base64_encoded_unified_file_id(file_id)
if is_unified_file_id:
unified_file_id = convert_b64_uid_to_unified_uid(file_id)
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split(
"llm_output_file_id,"
)[1].split(";")[0]
else:
# Fallback: keep original if we can't extract
provider_file_id = file_id
provider_file_id = file_id # Default to original
# Check if we have a mapping for this file ID
if model_file_id_mapping and model_id and file_id in model_file_id_mapping:
# Use the model-specific file ID from mapping
provider_file_id = (
model_file_id_mapping.get(file_id, {}).get(model_id)
or file_id
)
updated_content_item = content_item.copy()
updated_content_item["file_id"] = provider_file_id
updated_content.append(updated_content_item)
else:
updated_content.append(content_item)
# Check if this is a base64-encoded unified file ID without mapping
is_unified_file_id = _is_base64_encoded_unified_file_id(file_id)
if is_unified_file_id:
# Fallback: decode unified file ID
unified_file_id = convert_b64_uid_to_unified_uid(file_id)
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split(
"llm_output_file_id,"
)[1].split(";")[0]
updated_content_item = content_item.copy()
updated_content_item["file_id"] = provider_file_id
updated_content.append(updated_content_item)
else:
# Not a managed file, keep as-is
updated_content.append(content_item)
else:
updated_content.append(content_item)
else:
@ -506,6 +527,68 @@ def update_responses_input_with_model_file_ids(
return updated_input
def update_responses_tools_with_model_file_ids(
tools: Optional[List[Dict[str, Any]]],
model_id: Optional[str] = None,
model_file_id_mapping: Optional[Dict[str, Dict[str, str]]] = None,
) -> Optional[List[Dict[str, Any]]]:
"""
Updates responses API tools with provider-specific file IDs.
Handles code_interpreter tools with container.file_ids.
Args:
tools: The responses API tools parameter
model_id: The model ID to use for looking up provider-specific file IDs
model_file_id_mapping: Dictionary mapping litellm file IDs to provider file IDs
Format: {"litellm_file_id": {"model_id": "provider_file_id"}}
"""
if not tools or not isinstance(tools, list):
return tools
if not model_file_id_mapping or not model_id:
return tools
updated_tools = []
for tool in tools:
if not isinstance(tool, dict):
updated_tools.append(tool)
continue
updated_tool = tool.copy()
# Handle code_interpreter with container file_ids
if tool.get("type") == "code_interpreter":
container = tool.get("container")
if isinstance(container, dict):
container_file_ids = container.get("file_ids")
if isinstance(container_file_ids, list):
updated_file_ids = []
for file_id in container_file_ids:
if isinstance(file_id, str):
# Check if we have a mapping for this file ID
if file_id in model_file_id_mapping:
# Map to provider-specific file ID
provider_file_id = (
model_file_id_mapping.get(file_id, {}).get(model_id)
or file_id
)
updated_file_ids.append(provider_file_id)
else:
updated_file_ids.append(file_id)
else:
updated_file_ids.append(file_id)
# Update the tool with new file IDs
updated_container = container.copy()
updated_container["file_ids"] = updated_file_ids
updated_tool["container"] = updated_container
updated_tools.append(updated_tool)
return updated_tools
def extract_file_data(file_data: FileTypes) -> ExtractedFileData:
"""
Extracts and processes file data from various input formats.

36
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -3987,10 +3987,12 @@ class BedrockConverseMessagesProcessor:
assistant_parts=assistants_parts,
)
elif element["type"] == "text":
assistants_part = BedrockContentBlock(
text=element["text"]
)
assistants_parts.append(assistants_part)
# Skip completely empty strings to avoid blank content blocks
if element.get("text", "").strip():
assistants_part = BedrockContentBlock(
text=element["text"]
)
assistants_parts.append(assistants_part)
elif element["type"] == "image_url":
if isinstance(element["image_url"], dict):
image_url = element["image_url"]["url"]
@ -4015,9 +4017,12 @@ class BedrockConverseMessagesProcessor:
elif _assistant_content is not None and isinstance(
_assistant_content, str
):
assistant_content.append(
BedrockContentBlock(text=_assistant_content)
)
# Skip completely empty strings to avoid blank content blocks
if _assistant_content.strip():
assistant_content.append(
BedrockContentBlock(text=_assistant_content)
)
# If content is empty/whitespace, skip it (don't add a placeholder)
# Add cache point block for assistant string content
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
@ -4348,12 +4353,11 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
assistant_parts=assistants_parts,
)
elif element["type"] == "text":
# AWS Bedrock doesn't allow empty or whitespace-only text content, so use placeholder for empty strings
text_content = (
element["text"] if element["text"].strip() else "."
)
assistants_part = BedrockContentBlock(text=text_content)
assistants_parts.append(assistants_part)
# AWS Bedrock doesn't allow empty or whitespace-only text content
# Skip completely empty strings to avoid blank content blocks
if element.get("text", "").strip():
assistants_part = BedrockContentBlock(text=element["text"])
assistants_parts.append(assistants_part)
elif element["type"] == "image_url":
if isinstance(element["image_url"], dict):
image_url = element["image_url"]["url"]
@ -4376,9 +4380,9 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
assistants_parts.append(_cache_point_block)
assistant_content.extend(assistants_parts)
elif _assistant_content is not None and isinstance(_assistant_content, str):
# AWS Bedrock doesn't allow empty or whitespace-only text content, so use placeholder for empty strings
text_content = _assistant_content if _assistant_content.strip() else "."
assistant_content.append(BedrockContentBlock(text=text_content))
# Skip completely empty strings to avoid blank content blocks
if _assistant_content.strip():
assistant_content.append(BedrockContentBlock(text=_assistant_content))
# Add cache point block for assistant string content
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(

47
litellm/litellm_core_utils/redact_messages.py
View file

@ -130,6 +130,11 @@ def perform_redaction(model_call_details: dict, result):
def should_redact_message_logging(model_call_details: dict) -> bool:
"""
Determine if message logging should be redacted.
Priority order:
1. Dynamic parameter (turn_off_message_logging in request)
2. Headers (litellm-disable-message-redaction / litellm-enable-message-redaction)
3. Global setting (litellm.turn_off_message_logging)
"""
litellm_params = model_call_details.get("litellm_params", {})
@ -139,36 +144,36 @@ def should_redact_message_logging(model_call_details: dict) -> bool:
# Get headers from the metadata
request_headers = metadata.get("headers", {}) if isinstance(metadata, dict) else {}
possible_request_headers = [
# Check for headers that explicitly control redaction
if request_headers and bool(
request_headers.get("litellm-disable-message-redaction", False)
):
# User explicitly disabled redaction via header
return False
possible_enable_headers = [
"litellm-enable-message-redaction", # old header. maintain backwards compatibility
"x-litellm-enable-message-redaction", # new header
]
is_redaction_enabled_via_header = False
for header in possible_request_headers:
for header in possible_enable_headers:
if bool(request_headers.get(header, False)):
is_redaction_enabled_via_header = True
break
# check if user opted out of logging message/response to callbacks
if (
litellm.turn_off_message_logging is not True
and is_redaction_enabled_via_header is not True
and _get_turn_off_message_logging_from_dynamic_params(model_call_details)
is not True
):
return False
if request_headers and bool(
request_headers.get("litellm-disable-message-redaction", False)
):
return False
# user has OPTED OUT of message redaction
if _get_turn_off_message_logging_from_dynamic_params(model_call_details) is False:
return False
return True
# Priority 1: Check dynamic parameter first (if explicitly set)
dynamic_turn_off = _get_turn_off_message_logging_from_dynamic_params(model_call_details)
if dynamic_turn_off is not None:
# Dynamic parameter is explicitly set, use it
return dynamic_turn_off
# Priority 2: Check if header explicitly enables redaction
if is_redaction_enabled_via_header:
return True
# Priority 3: Fall back to global setting
return litellm.turn_off_message_logging is True
def redact_message_input_output_from_logging(

2
litellm/llms/a2a/chat/streaming_iterator.py
View file

@ -94,7 +94,7 @@ class A2AModelResponseIterator(BaseModelResponseIterator):
if state == "completed":
return "stop"
elif state == "failed":
return "error"
return "stop" # Map failed state to 'stop' (valid finish_reason)
# Check for [DONE] marker
if chunk.get("done") is True:

79
litellm/llms/a2a/chat/transformation.py
View file

@ -2,10 +2,9 @@
A2A Protocol Transformation for LiteLLM
"""
import uuid
from typing import Any, Dict, Iterator, List, Optional, Union, cast
from typing import Any, Dict, Iterator, List, Optional, Union
import httpx
from pydantic import BaseModel
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -27,6 +26,68 @@ class A2AConfig(BaseConfig):
Handles transformation between OpenAI and A2A JSON-RPC 2.0 formats.
"""
@staticmethod
def resolve_agent_config_from_registry(
model: str,
api_base: Optional[str],
api_key: Optional[str],
headers: Optional[Dict[str, Any]],
optional_params: Dict[str, Any],
) -> tuple[Optional[str], Optional[str], Optional[Dict[str, Any]]]:
"""
Resolve agent configuration from registry if model format is "a2a/<agent-name>".
Extracts agent name from model string and looks up configuration in the
agent registry (if available in proxy context).
Args:
model: Model string (e.g., "a2a/my-agent")
api_base: Explicit api_base (takes precedence over registry)
api_key: Explicit api_key (takes precedence over registry)
headers: Explicit headers (takes precedence over registry)
optional_params: Dict to merge additional litellm_params into
Returns:
Tuple of (api_base, api_key, headers) with registry values filled in
"""
# Extract agent name from model (e.g., "a2a/my-agent" -> "my-agent")
agent_name = model.split("/", 1)[1] if "/" in model else None
# Only lookup if agent name exists and some config is missing
if not agent_name or (api_base is not None and api_key is not None and headers is not None):
return api_base, api_key, headers
# Try registry lookup (only available in proxy context)
try:
from litellm.proxy.agent_endpoints.agent_registry import (
global_agent_registry,
)
agent = global_agent_registry.get_agent_by_name(agent_name)
if agent:
# Get api_base from agent card URL
if api_base is None and agent.agent_card_params:
api_base = agent.agent_card_params.get("url")
# Get api_key, headers, and other params from litellm_params
if agent.litellm_params:
if api_key is None:
api_key = agent.litellm_params.get("api_key")
if headers is None:
agent_headers = agent.litellm_params.get("headers")
if agent_headers:
headers = agent_headers
# Merge other litellm_params (timeout, max_retries, etc.)
for key, value in agent.litellm_params.items():
if key not in ["api_key", "api_base", "headers", "model"] and key not in optional_params:
optional_params[key] = value
except ImportError:
pass # Registry not available (not running in proxy context)
return api_base, api_key, headers
def get_supported_openai_params(self, model: str) -> List[str]:
"""Return list of supported OpenAI parameters"""
return [
@ -46,9 +107,14 @@ class A2AConfig(BaseConfig):
"""
Map OpenAI parameters to A2A parameters.
For A2A protocol, we don't need to map most parameters since
they're handled in the transform_request method.
For A2A protocol, we need to map the stream parameter so
transform_request can determine which JSON-RPC method to use.
"""
# Map stream parameter
for param, value in non_default_params.items():
if param == "stream" and value is True:
optional_params["stream"] = value
return optional_params
def validate_environment(
@ -160,8 +226,9 @@ class A2AConfig(BaseConfig):
# Build JSON-RPC 2.0 request
# For A2A protocol, the method is "message/send" for non-streaming
# and "message/stream" for streaming (handled by optional_params["stream"])
method = "message/stream" if optional_params.get("stream") else "message/send"
# and "message/stream" for streaming
stream = optional_params.get("stream", False)
method = "message/stream" if stream else "message/send"
request_data = {
"jsonrpc": "2.0",

20
litellm/llms/a2a/common_utils.py
View file

@ -113,6 +113,8 @@ def extract_text_from_a2a_response(
# 1. Direct message: {"result": {"kind": "message", "parts": [...]}}
# 2. Nested message: {"result": {"message": {"parts": [...]}}}
# 3. Task with artifacts: {"result": {"kind": "task", "artifacts": [{"parts": [...]}]}}
# 4. Task with status message: {"result": {"kind": "task", "status": {"message": {"parts": [...]}}}}
# 5. Streaming artifact-update: {"result": {"kind": "artifact-update", "artifact": {"parts": [...]}}}
# Check if result itself has parts (direct message)
if "parts" in result:
@ -123,7 +125,23 @@ def extract_text_from_a2a_response(
if message:
return extract_text_from_a2a_message(message, depth=0, max_depth=max_depth)
# Handle task result with artifacts
# Check for streaming artifact-update (singular artifact)
artifact = result.get("artifact")
if artifact and isinstance(artifact, dict):
return extract_text_from_a2a_message(
artifact, depth=0, max_depth=max_depth
)
# Check for task status message (common in Gemini A2A agents)
status = result.get("status", {})
if isinstance(status, dict):
status_message = status.get("message")
if status_message:
return extract_text_from_a2a_message(
status_message, depth=0, max_depth=max_depth
)
# Handle task result with artifacts (plural, array)
artifacts = result.get("artifacts", [])
if artifacts and len(artifacts) > 0:
first_artifact = artifacts[0]

72
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -34,6 +34,7 @@ from litellm.types.llms.openai import (
)
from litellm.types.utils import (
ChatCompletionMessageToolCall,
Choices,
GenericGuardrailAPIInputs,
ModelResponse,
)
@ -76,7 +77,8 @@ class AnthropicMessagesHandler(BaseTranslation):
chat_completion_compatible_request, tool_name_mapping = (
LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request=cast(AnthropicMessagesRequest, data)
# Use a shallow copy to avoid mutating request data (pop on litellm_metadata).
anthropic_message_request=cast(AnthropicMessagesRequest, data.copy())
)
)
@ -84,9 +86,9 @@ class AnthropicMessagesHandler(BaseTranslation):
texts_to_check: List[str] = []
images_to_check: List[str] = []
tools_to_check: List[ChatCompletionToolParam] = (
chat_completion_compatible_request.get("tools", [])
)
tools_to_check: List[
ChatCompletionToolParam
] = chat_completion_compatible_request.get("tools", [])
task_mappings: List[Tuple[int, Optional[int]]] = []
# Track (message_index, content_index) for each text
# content_index is None for string content, int for list content
@ -282,7 +284,10 @@ class AnthropicMessagesHandler(BaseTranslation):
if hasattr(content_block, "model_dump"):
block_dict = content_block.model_dump()
else:
block_dict = {"type": block_type, "text": getattr(content_block, "text", None)}
block_dict = {
"type": block_type,
"text": getattr(content_block, "text", None),
}
else:
continue
@ -358,30 +363,40 @@ class AnthropicMessagesHandler(BaseTranslation):
"""
has_ended = self._check_streaming_has_ended(responses_so_far)
if has_ended:
# build the model response from the responses_so_far
model_response = cast(
ModelResponse,
AnthropicPassthroughLoggingHandler._build_complete_streaming_response(
all_chunks=responses_so_far,
litellm_logging_obj=cast("LiteLLMLoggingObj", litellm_logging_obj),
model="",
),
built_response = AnthropicPassthroughLoggingHandler._build_complete_streaming_response(
all_chunks=responses_so_far,
litellm_logging_obj=cast("LiteLLMLoggingObj", litellm_logging_obj),
model="",
)
tool_calls_list = cast(Optional[List[ChatCompletionMessageToolCall]], model_response.choices[0].message.tool_calls) # type: ignore
string_so_far = model_response.choices[0].message.content # type: ignore
guardrail_inputs = GenericGuardrailAPIInputs()
if string_so_far:
guardrail_inputs["texts"] = [string_so_far]
if tool_calls_list:
guardrail_inputs["tool_calls"] = tool_calls_list
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid
inputs=guardrail_inputs,
request_data={},
input_type="response",
logging_obj=litellm_logging_obj,
)
# Check if model_response is valid and has choices before accessing
if (
built_response is not None
and hasattr(built_response, "choices")
and built_response.choices
):
model_response = cast(ModelResponse, built_response)
first_choice = cast(Choices, model_response.choices[0])
tool_calls_list = cast(
Optional[List[ChatCompletionMessageToolCall]],
first_choice.message.tool_calls,
)
string_so_far = first_choice.message.content
guardrail_inputs = GenericGuardrailAPIInputs()
if string_so_far:
guardrail_inputs["texts"] = [string_so_far]
if tool_calls_list:
guardrail_inputs["tool_calls"] = tool_calls_list
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail( # allow rejecting the response, if invalid
inputs=guardrail_inputs,
request_data={},
input_type="response",
logging_obj=litellm_logging_obj,
)
else:
verbose_proxy_logger.debug("Skipping output guardrail - model response has no choices")
return responses_so_far
string_so_far = self.get_streaming_string_so_far(responses_so_far)
@ -648,7 +663,10 @@ class AnthropicMessagesHandler(BaseTranslation):
if isinstance(content_block, dict):
if content_block.get("type") == "text":
cast(Dict[str, Any], content_block)["text"] = guardrail_response
elif hasattr(content_block, "type") and getattr(content_block, "type", None) == "text":
elif (
hasattr(content_block, "type")
and getattr(content_block, "type", None) == "text"
):
# Update Pydantic object's text attribute
if hasattr(content_block, "text"):
content_block.text = guardrail_response

28
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -939,22 +939,8 @@ class LiteLLMAnthropicMessagesAdapter:
self,
choices: List[Choices],
tool_name_mapping: Optional[Dict[str, str]] = None,
) -> List[
Union[
AnthropicResponseContentBlockText,
AnthropicResponseContentBlockToolUse,
AnthropicResponseContentBlockThinking,
AnthropicResponseContentBlockRedactedThinking,
]
]:
new_content: List[
Union[
AnthropicResponseContentBlockText,
AnthropicResponseContentBlockToolUse,
AnthropicResponseContentBlockThinking,
AnthropicResponseContentBlockRedactedThinking,
]
] = []
) -> List[Dict[str, Any]]:
new_content: List[Dict[str, Any]] = []
for choice in choices:
# Handle thinking blocks first
if (
@ -978,7 +964,7 @@ class LiteLLMAnthropicMessagesAdapter:
if signature_value is not None
else None
),
)
).model_dump()
)
elif thinking_block.get("type") == "redacted_thinking":
data_value = thinking_block.get("data", "")
@ -986,7 +972,7 @@ class LiteLLMAnthropicMessagesAdapter:
AnthropicResponseContentBlockRedactedThinking(
type="redacted_thinking",
data=str(data_value) if data_value is not None else "",
)
).model_dump()
)
# Handle reasoning_content when thinking_blocks is not present
elif (
@ -998,7 +984,7 @@ class LiteLLMAnthropicMessagesAdapter:
type="thinking",
thinking=str(choice.message.reasoning_content),
signature=None,
)
).model_dump()
)
# Handle text content
@ -1006,7 +992,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_content.append(
AnthropicResponseContentBlockText(
type="text", text=choice.message.content
)
).model_dump()
)
# Handle tool calls (in parallel to text content)
if (
@ -1044,7 +1030,7 @@ class LiteLLMAnthropicMessagesAdapter:
tool_use_block.provider_specific_fields = (
provider_specific_fields
)
new_content.append(tool_use_block)
new_content.append(tool_use_block.model_dump())
return new_content

100
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -30,6 +30,8 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.types.llms.bedrock import *
from ..common_utils import is_claude_4_5_on_bedrock
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
@ -306,9 +308,7 @@ class AmazonConverseConfig(BaseConfig):
return "nova-2-lite" in model_without_region
def _map_web_search_options(
self,
web_search_options: dict,
model: str
self, web_search_options: dict, model: str
) -> Optional[BedrockToolBlock]:
"""
Map web_search_options to Nova grounding systemTool.
@ -634,7 +634,7 @@ class AmazonConverseConfig(BaseConfig):
Filtered list of beta headers
"""
filtered_betas = []
# 1. Filter out beta headers that are universally unsupported on Bedrock Converse
for beta in beta_list:
should_keep = True
@ -642,10 +642,10 @@ class AmazonConverseConfig(BaseConfig):
if unsupported_pattern in beta.lower():
should_keep = False
break
if should_keep:
filtered_betas.append(beta)
return filtered_betas
def _separate_computer_use_tools(
@ -808,11 +808,11 @@ class AmazonConverseConfig(BaseConfig):
if param == "web_search_options" and isinstance(value, dict):
# Note: we use `isinstance(value, dict)` instead of `value and isinstance(value, dict)`
# because empty dict {} is falsy but is a valid way to enable Nova grounding
grounding_tool = self._map_web_search_options(value, model)
if grounding_tool is not None:
optional_params = self._add_tools_to_optional_params(
optional_params=optional_params, tools=[grounding_tool]
)
grounding_tool = self._map_web_search_options(value, model)
if grounding_tool is not None:
optional_params = self._add_tools_to_optional_params(
optional_params=optional_params, tools=[grounding_tool]
)
# Only update thinking tokens for non-GPT-OSS models and non-Nova-Lite-2 models
# Nova Lite 2 handles token budgeting differently through reasoningConfig
@ -926,6 +926,7 @@ class AmazonConverseConfig(BaseConfig):
ChatCompletionAssistantMessage,
],
block_type: Literal["system"],
model: Optional[str] = None,
) -> Optional[SystemContentBlock]:
pass
@ -939,6 +940,7 @@ class AmazonConverseConfig(BaseConfig):
ChatCompletionAssistantMessage,
],
block_type: Literal["content_block"],
model: Optional[str] = None,
) -> Optional[ContentBlock]:
pass
@ -951,16 +953,26 @@ class AmazonConverseConfig(BaseConfig):
ChatCompletionAssistantMessage,
],
block_type: Literal["system", "content_block"],
model: Optional[str] = None,
) -> Optional[Union[SystemContentBlock, ContentBlock]]:
if message_block.get("cache_control", None) is None:
cache_control = message_block.get("cache_control", None)
if cache_control is None:
return None
cache_point = CachePointBlock(type="default")
if isinstance(cache_control, dict) and "ttl" in cache_control:
ttl = cache_control["ttl"]
if ttl in ["5m", "1h"] and model is not None:
if is_claude_4_5_on_bedrock(model):
cache_point["ttl"] = ttl
if block_type == "system":
return SystemContentBlock(cachePoint=CachePointBlock(type="default"))
return SystemContentBlock(cachePoint=cache_point)
else:
return ContentBlock(cachePoint=CachePointBlock(type="default"))
return ContentBlock(cachePoint=cache_point)
def _transform_system_message(
self, messages: List[AllMessageValues]
self, messages: List[AllMessageValues], model: Optional[str] = None
) -> Tuple[List[AllMessageValues], List[SystemContentBlock]]:
system_prompt_indices = []
system_content_blocks: List[SystemContentBlock] = []
@ -972,7 +984,7 @@ class AmazonConverseConfig(BaseConfig):
SystemContentBlock(text=message["content"])
)
cache_block = self._get_cache_point_block(
message, block_type="system"
message, block_type="system", model=model
)
if cache_block:
system_content_blocks.append(cache_block)
@ -983,7 +995,7 @@ class AmazonConverseConfig(BaseConfig):
SystemContentBlock(text=m["text"])
)
cache_block = self._get_cache_point_block(
m, block_type="system"
m, block_type="system", model=model
)
if cache_block:
system_content_blocks.append(cache_block)
@ -1137,13 +1149,13 @@ class AmazonConverseConfig(BaseConfig):
if beta not in seen:
unique_betas.append(beta)
seen.add(beta)
# Filter out unsupported beta headers for Bedrock Converse API
filtered_betas = self._filter_unsupported_beta_headers_for_bedrock(
model=model,
beta_list=unique_betas,
)
additional_request_params["anthropic_beta"] = filtered_betas
return bedrock_tools, anthropic_beta_list
@ -1196,9 +1208,11 @@ class AmazonConverseConfig(BaseConfig):
)
# Prepare and separate parameters
inference_params, additional_request_params, request_metadata = self._prepare_request_params(
optional_params, model
)
(
inference_params,
additional_request_params,
request_metadata,
) = self._prepare_request_params(optional_params, model)
original_tools = inference_params.pop("tools", [])
@ -1250,7 +1264,9 @@ class AmazonConverseConfig(BaseConfig):
litellm_params: dict,
headers: Optional[dict] = None,
) -> RequestObject:
messages, system_content_blocks = self._transform_system_message(messages)
messages, system_content_blocks = self._transform_system_message(
messages, model=model
)
# Convert last user message to guarded_text if guardrailConfig is present
messages = self._convert_consecutive_user_messages_to_guarded_text(
@ -1306,7 +1322,9 @@ class AmazonConverseConfig(BaseConfig):
litellm_params: dict,
headers: Optional[dict] = None,
) -> RequestObject:
messages, system_content_blocks = self._transform_system_message(messages)
messages, system_content_blocks = self._transform_system_message(
messages, model=model
)
# Convert last user message to guarded_text if guardrailConfig is present
messages = self._convert_consecutive_user_messages_to_guarded_text(
@ -1484,7 +1502,9 @@ class AmazonConverseConfig(BaseConfig):
return message, returned_finish_reason
def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[
def _translate_message_content(
self, content_blocks: List[ContentBlock]
) -> Tuple[
str,
List[ChatCompletionToolCallChunk],
Optional[List[BedrockConverseReasoningContentBlock]],
@ -1501,9 +1521,9 @@ class AmazonConverseConfig(BaseConfig):
"""
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
None
)
reasoningContentBlocks: Optional[
List[BedrockConverseReasoningContentBlock]
] = None
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
for idx, content in enumerate(content_blocks):
"""
@ -1557,7 +1577,7 @@ class AmazonConverseConfig(BaseConfig):
return content_str, tools, reasoningContentBlocks, citationsContentBlocks
def _transform_response( # noqa: PLR0915
def _transform_response( # noqa: PLR0915
self,
model: str,
response: httpx.Response,
@ -1630,9 +1650,9 @@ class AmazonConverseConfig(BaseConfig):
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
content_str = ""
tools: List[ChatCompletionToolCallChunk] = []
reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
None
)
reasoningContentBlocks: Optional[
List[BedrockConverseReasoningContentBlock]
] = None
citationsContentBlocks: Optional[List[CitationsContentBlock]] = None
if message is not None:
@ -1651,15 +1671,17 @@ class AmazonConverseConfig(BaseConfig):
provider_specific_fields["citationsContent"] = citationsContentBlocks
if provider_specific_fields:
chat_completion_message["provider_specific_fields"] = provider_specific_fields
chat_completion_message[
"provider_specific_fields"
] = provider_specific_fields
if reasoningContentBlocks is not None:
chat_completion_message["reasoning_content"] = (
self._transform_reasoning_content(reasoningContentBlocks)
)
chat_completion_message["thinking_blocks"] = (
self._transform_thinking_blocks(reasoningContentBlocks)
)
chat_completion_message[
"reasoning_content"
] = self._transform_reasoning_content(reasoningContentBlocks)
chat_completion_message[
"thinking_blocks"
] = self._transform_thinking_blocks(reasoningContentBlocks)
chat_completion_message["content"] = content_str
if (
json_mode is True

33
litellm/llms/bedrock/common_utils.py
View file

@ -446,6 +446,29 @@ def get_bedrock_base_model(model: str) -> str:
return model
def is_claude_4_5_on_bedrock(model: str) -> bool:
"""
Check if the model is a Claude 4.5 model on Bedrock.
Claude 4.5 models support prompt caching with '5m' and '1h' TTL on Bedrock.
"""
model_lower = model.lower()
claude_4_5_patterns = [
"sonnet-4.5",
"sonnet_4.5",
"sonnet-4-5",
"sonnet_4_5",
"haiku-4.5",
"haiku_4.5",
"haiku-4-5",
"haiku_4_5",
"opus-4.5",
"opus_4.5",
"opus-4-5",
"opus_4_5",
]
return any(pattern in model_lower for pattern in claude_4_5_patterns)
# Import after standalone functions to avoid circular imports
from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter
@ -815,21 +838,23 @@ def get_anthropic_beta_from_headers(headers: dict) -> List[str]:
# If it's already a list, return it
if isinstance(anthropic_beta_header, list):
return anthropic_beta_header
# Try to parse as JSON array first (e.g., '["interleaved-thinking-2025-05-14", "claude-code-20250219"]')
if isinstance(anthropic_beta_header, str):
anthropic_beta_header = anthropic_beta_header.strip()
if anthropic_beta_header.startswith("[") and anthropic_beta_header.endswith("]"):
if anthropic_beta_header.startswith("[") and anthropic_beta_header.endswith(
"]"
):
try:
parsed = json.loads(anthropic_beta_header)
if isinstance(parsed, list):
return [str(beta).strip() for beta in parsed]
except json.JSONDecodeError:
pass # Fall through to comma-separated parsing
# Fall back to comma-separated values
return [beta.strip() for beta in anthropic_beta_header.split(",")]
return []

128
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -23,7 +23,10 @@ from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
)
from litellm.llms.bedrock.common_utils import get_anthropic_beta_from_headers
from litellm.llms.bedrock.common_utils import (
get_anthropic_beta_from_headers,
is_claude_4_5_on_bedrock,
)
from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
@ -54,7 +57,7 @@ class AmazonAnthropicClaudeMessagesConfig(
# These will be filtered out to prevent 400 "invalid beta flag" errors
UNSUPPORTED_BEDROCK_INVOKE_BETA_PATTERNS = [
"advanced-tool-use", # Bedrock Invoke doesn't support advanced-tool-use beta headers
"prompt-caching-scope"
"prompt-caching-scope",
]
def __init__(self, **kwargs):
@ -116,15 +119,22 @@ class AmazonAnthropicClaudeMessagesConfig(
)
def _remove_ttl_from_cache_control(
self, anthropic_messages_request: Dict
self, anthropic_messages_request: Dict, model: Optional[str] = None
) -> None:
"""
Remove `ttl` field from cache_control in messages.
Bedrock doesn't support the ttl field in cache_control.
Update: Bedock supports `5m` and `1h` for Claude 4.5 models.
Args:
anthropic_messages_request: The request dictionary to modify in-place
model: The model name to check if it supports ttl
"""
is_claude_4_5 = False
if model:
is_claude_4_5 = self._is_claude_4_5_on_bedrock(model)
if "messages" in anthropic_messages_request:
for message in anthropic_messages_request["messages"]:
if isinstance(message, dict) and "content" in message:
@ -133,7 +143,14 @@ class AmazonAnthropicClaudeMessagesConfig(
for item in content:
if isinstance(item, dict) and "cache_control" in item:
cache_control = item["cache_control"]
if isinstance(cache_control, dict) and "ttl" in cache_control:
if (
isinstance(cache_control, dict)
and "ttl" in cache_control
):
ttl = cache_control["ttl"]
if is_claude_4_5 and ttl in ["5m", "1h"]:
continue
cache_control.pop("ttl", None)
def _supports_extended_thinking_on_bedrock(self, model: str) -> bool:
@ -155,10 +172,18 @@ class AmazonAnthropicClaudeMessagesConfig(
# Supported models on Bedrock for extended thinking
supported_patterns = [
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5", # Opus 4.5
"opus-4.1", "opus_4.1", "opus-4-1", "opus_4_1", # Opus 4.1
"opus-4", "opus_4", # Opus 4
"sonnet-4", "sonnet_4", # Sonnet 4
"opus-4.5",
"opus_4.5",
"opus-4-5",
"opus_4_5", # Opus 4.5
"opus-4.1",
"opus_4.1",
"opus-4-1",
"opus_4_1", # Opus 4.1
"opus-4",
"opus_4", # Opus 4
"sonnet-4",
"sonnet_4", # Sonnet 4
]
return any(pattern in model_lower for pattern in supported_patterns)
@ -175,10 +200,27 @@ class AmazonAnthropicClaudeMessagesConfig(
"""
model_lower = model.lower()
opus_4_5_patterns = [
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5",
"opus-4.5",
"opus_4.5",
"opus-4-5",
"opus_4_5",
]
return any(pattern in model_lower for pattern in opus_4_5_patterns)
def _is_claude_4_5_on_bedrock(self, model: str) -> bool:
"""
Check if the model is Claude 4.5 on Bedrock.
Claude Sonnet 4.5, Haiku 4.5, and Opus 4.5 support 1-hour prompt caching.
Args:
model: The model name
Returns:
True if the model is Claude 4.5
"""
return is_claude_4_5_on_bedrock(model)
def _supports_tool_search_on_bedrock(self, model: str) -> bool:
"""
Check if the model supports tool search on Bedrock.
@ -199,9 +241,15 @@ class AmazonAnthropicClaudeMessagesConfig(
# Supported models for tool search on Bedrock
supported_patterns = [
# Opus 4.5
"opus-4.5", "opus_4.5", "opus-4-5", "opus_4_5",
"opus-4.5",
"opus_4.5",
"opus-4-5",
"opus_4_5",
# Sonnet 4.5
"sonnet-4.5", "sonnet_4.5", "sonnet-4-5", "sonnet_4_5",
"sonnet-4.5",
"sonnet_4.5",
"sonnet-4-5",
"sonnet_4_5",
]
return any(pattern in model_lower for pattern in supported_patterns)
@ -238,8 +286,7 @@ class AmazonAnthropicClaudeMessagesConfig(
beta_headers_to_remove.add(beta)
has_advanced_tool_use = True
break
# 2. Filter out extended thinking headers for models that don't support them
extended_thinking_patterns = [
"extended-thinking",
@ -263,7 +310,6 @@ class AmazonAnthropicClaudeMessagesConfig(
beta_set.add("tool-search-tool-2025-10-19")
beta_set.add("tool-examples-2025-10-29")
def _get_tool_search_beta_header_for_bedrock(
self,
model: str,
@ -290,7 +336,9 @@ class AmazonAnthropicClaudeMessagesConfig(
input_examples_used: Whether input examples are used
beta_set: The set of beta headers to modify in-place
"""
if tool_search_used and not (programmatic_tool_calling_used or input_examples_used):
if tool_search_used and not (
programmatic_tool_calling_used or input_examples_used
):
beta_set.discard(ANTHROPIC_TOOL_SEARCH_BETA_HEADER)
if "opus-4" in model.lower() or "opus_4" in model.lower():
beta_set.add("tool-search-tool-2025-10-19")
@ -302,13 +350,13 @@ class AmazonAnthropicClaudeMessagesConfig(
) -> None:
"""
Convert Anthropic output_format to inline schema in message content.
Bedrock Invoke doesn't support the output_format parameter, so we embed
the schema directly into the user message content as text instructions.
This approach adds the schema to the last user message, instructing the model
to respond in the specified JSON format.
Args:
output_format: The output_format dict with 'type' and 'schema'
anthropic_messages_request: The request dict to modify in-place
@ -321,35 +369,32 @@ class AmazonAnthropicClaudeMessagesConfig(
schema = output_format.get("schema")
if not schema:
return
# Get messages from the request
messages = anthropic_messages_request.get("messages", [])
if not messages:
return
# Find the last user message
last_user_message_idx = None
for idx in range(len(messages) - 1, -1, -1):
if messages[idx].get("role") == "user":
last_user_message_idx = idx
break
if last_user_message_idx is None:
return
last_user_message = messages[last_user_message_idx]
content = last_user_message.get("content", [])
# Ensure content is a list
if isinstance(content, str):
content = [{"type": "text", "text": content}]
last_user_message["content"] = content
# Add schema as text content to the message
schema_text = {
"type": "text",
"text": json.dumps(schema)
}
schema_text = {"type": "text", "text": json.dumps(schema)}
content.append(schema_text)
def transform_anthropic_messages_request(
@ -374,9 +419,9 @@ class AmazonAnthropicClaudeMessagesConfig(
# 1. anthropic_version is required for all claude models
if "anthropic_version" not in anthropic_messages_request:
anthropic_messages_request["anthropic_version"] = (
self.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION
)
anthropic_messages_request[
"anthropic_version"
] = self.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION
# 2. `stream` is not allowed in request body for bedrock invoke
if "stream" in anthropic_messages_request:
@ -386,8 +431,10 @@ class AmazonAnthropicClaudeMessagesConfig(
if "model" in anthropic_messages_request:
anthropic_messages_request.pop("model", None)
# 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it)
self._remove_ttl_from_cache_control(anthropic_messages_request)
# 4. Remove `ttl` field from cache_control in messages (Bedrock doesn't support it for older models)
self._remove_ttl_from_cache_control(
anthropic_messages_request=anthropic_messages_request, model=model
)
# 5. Convert `output_format` to inline schema (Bedrock invoke doesn't support output_format)
output_format = anthropic_messages_request.pop("output_format", None)
@ -396,14 +443,14 @@ class AmazonAnthropicClaudeMessagesConfig(
output_format=output_format,
anthropic_messages_request=anthropic_messages_request,
)
# 6. AUTO-INJECT beta headers based on features used
anthropic_model_info = AnthropicModelInfo()
tools = anthropic_messages_optional_request_params.get("tools")
messages_typed = cast(List[AllMessageValues], messages)
tool_search_used = anthropic_model_info.is_tool_search_used(tools)
programmatic_tool_calling_used = anthropic_model_info.is_programmatic_tool_calling_used(
tools
programmatic_tool_calling_used = (
anthropic_model_info.is_programmatic_tool_calling_used(tools)
)
input_examples_used = anthropic_model_info.is_input_examples_used(tools)
@ -436,8 +483,7 @@ class AmazonAnthropicClaudeMessagesConfig(
if beta_set:
anthropic_messages_request["anthropic_beta"] = list(beta_set)
return anthropic_messages_request
def get_async_streaming_response_iterator(
@ -455,7 +501,7 @@ class AmazonAnthropicClaudeMessagesConfig(
)
# Convert decoded Bedrock events to Server-Sent Events expected by Anthropic clients.
return self.bedrock_sse_wrapper(
completion_stream=completion_stream,
completion_stream=completion_stream,
litellm_logging_obj=litellm_logging_obj,
request_body=request_body,
)
@ -474,14 +520,14 @@ class AmazonAnthropicClaudeMessagesConfig(
from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
BaseAnthropicMessagesStreamingIterator,
)
handler = BaseAnthropicMessagesStreamingIterator(
litellm_logging_obj=litellm_logging_obj,
request_body=request_body,
)
async for chunk in handler.async_sse_wrapper(completion_stream):
yield chunk
class AmazonAnthropicClaudeMessagesStreamDecoder(AWSEventStreamDecoder):

4
litellm/llms/fireworks_ai/chat/transformation.py
View file

@ -236,6 +236,10 @@ class FireworksAIConfig(OpenAIGPTConfig):
disable_add_transform_inline_image_block=disable_add_transform_inline_image_block,
)
filter_value_from_dict(cast(dict, message), "cache_control")
# Remove fields not permitted by FireworksAI that may cause:
# "Not permitted, field: 'messages[n].provider_specific_fields'"
if isinstance(message, dict) and "provider_specific_fields" in message:
cast(dict, message).pop("provider_specific_fields", None)
return messages

7
litellm/llms/gemini/files/transformation.py
View file

@ -210,7 +210,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
We expect file_id to be the URI (e.g. https://generativelanguage.googleapis.com/v1beta/files/...)
as returned by the upload response.
"""
api_key = litellm_params.get("api_key")
api_key = litellm_params.get("api_key") or self.get_api_key()
if not api_key:
raise ValueError("api_key is required")
@ -222,7 +222,8 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
api_base = api_base.rstrip("/")
url = "{}/v1beta/{}?key={}".format(api_base, file_id, api_key)
return url, {"Content-Type": "application/json"}
# Return empty params dict - API key is already in URL, no query params needed
return url, {}
def transform_retrieve_file_response(
self,
@ -299,7 +300,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
# Extract the file path from full URI
file_name = file_id.split("/v1beta/")[-1]
else:
file_name = file_id
file_name = file_id if file_id.startswith("files/") else f"files/{file_id}"
# Construct the delete URL
url = f"{api_base}/v1beta/{file_name}"

28
litellm/llms/gigachat/chat/transformation.py
View file

@ -386,33 +386,7 @@ class GigaChatConfig(BaseConfig):
transformed.append(message)
# Collapse consecutive user messages
return self._collapse_user_messages(transformed)
def _collapse_user_messages(self, messages: List[dict]) -> List[dict]:
"""Collapse consecutive user messages into one."""
collapsed: List[dict] = []
prev_user_msg: Optional[dict] = None
content_parts: List[str] = []
for msg in messages:
if msg.get("role") == "user" and prev_user_msg is not None:
content_parts.append(msg.get("content", ""))
else:
if content_parts and prev_user_msg:
prev_user_msg["content"] = "\n".join(
[prev_user_msg.get("content", "")] + content_parts
)
content_parts = []
collapsed.append(msg)
prev_user_msg = msg if msg.get("role") == "user" else None
if content_parts and prev_user_msg:
prev_user_msg["content"] = "\n".join(
[prev_user_msg.get("content", "")] + content_parts
)
return collapsed
return transformed
def transform_response(
self,

51
litellm/llms/github_copilot/chat/transformation.py
View file

@ -1,11 +1,16 @@
from typing import Any, Optional, Tuple, cast, List
from typing import List, Optional, Tuple
from litellm.exceptions import AuthenticationError
from litellm.llms.openai.openai import OpenAIConfig
from litellm.types.llms.openai import AllMessageValues
from ..authenticator import Authenticator
from ..common_utils import GetAPIKeyError, GITHUB_COPILOT_API_BASE
from ..common_utils import (
GITHUB_COPILOT_API_BASE,
GetAPIKeyError,
get_copilot_default_headers,
)
class GithubCopilotConfig(OpenAIConfig):
@ -25,9 +30,7 @@ class GithubCopilotConfig(OpenAIConfig):
api_key: Optional[str],
custom_llm_provider: str,
) -> Tuple[Optional[str], Optional[str], str]:
dynamic_api_base = (
self.authenticator.get_api_base() or GITHUB_COPILOT_API_BASE
)
dynamic_api_base = self.authenticator.get_api_base() or GITHUB_COPILOT_API_BASE
try:
dynamic_api_key = self.authenticator.get_api_key()
except GetAPIKeyError as e:
@ -45,14 +48,24 @@ class GithubCopilotConfig(OpenAIConfig):
):
import litellm
disable_copilot_system_to_assistant = (
litellm.disable_copilot_system_to_assistant
)
if not disable_copilot_system_to_assistant:
for message in messages:
if "role" in message and message["role"] == "system":
cast(Any, message)["role"] = "assistant"
return messages
# Check if system-to-assistant conversion is disabled
if litellm.disable_copilot_system_to_assistant:
# GitHub Copilot API now supports system prompts for all models (Claude, GPT, etc.)
# No conversion needed - just return messages as-is
return messages
# Default behavior: convert system messages to assistant for compatibility
transformed_messages = []
for message in messages:
if message.get("role") == "system":
# Convert system message to assistant message
transformed_message = message.copy()
transformed_message["role"] = "assistant"
transformed_messages.append(transformed_message)
else:
transformed_messages.append(message)
return transformed_messages
def validate_environment(
self,
@ -69,6 +82,14 @@ class GithubCopilotConfig(OpenAIConfig):
headers, model, messages, optional_params, litellm_params, api_key, api_base
)
# Add Copilot-specific headers (editor-version, user-agent, etc.)
try:
copilot_api_key = self.authenticator.get_api_key()
copilot_headers = get_copilot_default_headers(copilot_api_key)
validated_headers = {**copilot_headers, **validated_headers}
except GetAPIKeyError:
pass # Will be handled later in the request flow
# Add X-Initiator header based on message roles
initiator = self._determine_initiator(messages)
validated_headers["X-Initiator"] = initiator
@ -87,7 +108,7 @@ class GithubCopilotConfig(OpenAIConfig):
For other models, returns standard OpenAI parameters (which may include reasoning_effort for o-series models).
"""
from litellm.utils import supports_reasoning
# Get base OpenAI parameters
base_params = super().get_supported_openai_params(model)
@ -118,7 +139,7 @@ class GithubCopilotConfig(OpenAIConfig):
"""
Check if any message contains vision content (images).
Returns True if any message has content with vision-related types, otherwise False.
Checks for:
- image_url content type (OpenAI format)
- Content items with type 'image_url'

21
litellm/llms/openai/chat/guardrail_translation/handler.py
View file

@ -21,7 +21,13 @@ from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.main import stream_chunk_builder
from litellm.types.llms.openai import ChatCompletionToolParam
from litellm.types.utils import Choices, GenericGuardrailAPIInputs, ModelResponse, ModelResponseStream, StreamingChoices
from litellm.types.utils import (
Choices,
GenericGuardrailAPIInputs,
ModelResponse,
ModelResponseStream,
StreamingChoices,
)
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
@ -80,9 +86,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if tool_calls_to_check:
inputs["tool_calls"] = tool_calls_to_check # type: ignore
if messages:
inputs["structured_messages"] = (
messages # pass the openai /chat/completions messages to the guardrail, as-is
)
inputs[
"structured_messages"
] = messages # pass the openai /chat/completions messages to the guardrail, as-is
# Pass tools (function definitions) to the guardrail
tools = data.get("tools")
if tools:
@ -362,14 +368,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
# check if the stream has ended
has_stream_ended = False
for chunk in responses_so_far:
if chunk.choices[0].finish_reason is not None:
if chunk.choices and chunk.choices[0].finish_reason is not None:
has_stream_ended = True
break
if has_stream_ended:
# convert to model response
model_response = cast(
ModelResponse, stream_chunk_builder(chunks=responses_so_far, logging_obj=litellm_logging_obj)
ModelResponse,
stream_chunk_builder(
chunks=responses_so_far, logging_obj=litellm_logging_obj
),
)
# run process_output_response
await self.process_output_response(

75
litellm/llms/openai/common_utils.py
View file

@ -15,14 +15,12 @@ if TYPE_CHECKING:
from aiohttp import ClientSession
import litellm
from litellm._logging import verbose_logger
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
AsyncHTTPHandler,
get_ssl_configuration,
)
from litellm.types.utils import LlmProviders
class OpenAIError(BaseLLMException):
@ -205,67 +203,30 @@ class BaseOpenAILLM:
if litellm.aclient_session is not None:
return litellm.aclient_session
# Use the global cached client system to prevent memory leaks (issue #14540)
# This routes through get_async_httpx_client() which provides TTL-based caching
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
# Get unified SSL configuration
ssl_config = get_ssl_configuration()
try:
# Get SSL config and include in params for proper cache key
ssl_config = get_ssl_configuration()
params = {"ssl_verify": ssl_config} if ssl_config is not None else {}
params["disable_aiohttp_transport"] = litellm.disable_aiohttp_transport
# Get a cached AsyncHTTPHandler which manages the httpx.AsyncClient
cached_handler = get_async_httpx_client(
llm_provider=LlmProviders.OPENAI, # Cache key includes provider
params=params, # Include SSL config in cache key
return httpx.AsyncClient(
verify=ssl_config,
transport=AsyncHTTPHandler._create_async_transport(
ssl_context=ssl_config
if isinstance(ssl_config, ssl.SSLContext)
else None,
ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
shared_session=shared_session,
)
# Return the underlying httpx client from the handler
return cached_handler.client
except (ImportError, AttributeError, KeyError) as e:
# Fallback to creating a client directly if caching system unavailable
# This preserves backwards compatibility
verbose_logger.debug(
f"Client caching unavailable ({type(e).__name__}), using direct client creation"
)
ssl_config = get_ssl_configuration()
return httpx.AsyncClient(
verify=ssl_config,
transport=AsyncHTTPHandler._create_async_transport(
ssl_context=ssl_config
if isinstance(ssl_config, ssl.SSLContext)
else None,
ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
shared_session=shared_session,
),
follow_redirects=True,
)
),
follow_redirects=True,
)
@staticmethod
def _get_sync_http_client() -> Optional[httpx.Client]:
if litellm.client_session is not None:
return litellm.client_session
# Use the global cached client system to prevent memory leaks (issue #14540)
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
# Get unified SSL configuration
ssl_config = get_ssl_configuration()
try:
# Get SSL config and include in params for proper cache key
ssl_config = get_ssl_configuration()
params = {"ssl_verify": ssl_config} if ssl_config is not None else None
# Get a cached HTTPHandler which manages the httpx.Client
cached_handler = _get_httpx_client(params=params)
# Return the underlying httpx client from the handler
return cached_handler.client
except (ImportError, AttributeError, KeyError) as e:
# Fallback to creating a client directly if caching system unavailable
verbose_logger.debug(
f"Client caching unavailable ({type(e).__name__}), using direct client creation"
)
ssl_config = get_ssl_configuration()
return httpx.Client(
verify=ssl_config,
follow_redirects=True,
)
return httpx.Client(
verify=ssl_config,
follow_redirects=True,
)

80
litellm/llms/openai/realtime/handler.py
View file

@ -16,6 +16,62 @@ from ..openai import OpenAIChatCompletion
class OpenAIRealtime(OpenAIChatCompletion):
"""
Base handler for OpenAI-compatible realtime WebSocket connections.
Subclasses can override template methods to customize:
- _get_default_api_base(): Default API base URL
- _get_additional_headers(): Extra headers beyond Authorization
- _get_ssl_config(): SSL configuration for WebSocket connection
"""
def _get_default_api_base(self) -> str:
"""
Get the default API base URL for this provider.
Override this in subclasses to set provider-specific defaults.
"""
return "https://api.openai.com/"
def _get_additional_headers(self, api_key: str) -> dict:
"""
Get additional headers beyond Authorization.
Override this in subclasses to customize headers (e.g., remove OpenAI-Beta).
Args:
api_key: API key for authentication
Returns:
Dictionary of additional headers
"""
return {
"Authorization": f"Bearer {api_key}",
"OpenAI-Beta": "realtime=v1",
}
def _get_ssl_config(self, url: str) -> Any:
"""
Get SSL configuration for WebSocket connection.
Override this in subclasses to customize SSL behavior.
Args:
url: WebSocket URL (ws:// or wss://)
Returns:
SSL configuration (None, True, or SSLContext)
"""
if url.startswith("ws://"):
return None
# Use the shared SSL context which respects custom CA certs and SSL settings
ssl_config = get_shared_realtime_ssl_context()
# If ssl_config is False (ssl_verify=False), websockets library needs True instead
# to establish connection without verification (False would fail)
if ssl_config is False:
return True
return ssl_config
def _construct_url(self, api_base: str, query_params: RealtimeQueryParams) -> str:
"""
Construct the backend websocket URL with all query parameters (including 'model').
@ -45,8 +101,9 @@ class OpenAIRealtime(OpenAIChatCompletion):
):
import websockets
from websockets.asyncio.client import ClientConnection
if api_base is None:
api_base = "https://api.openai.com/"
api_base = self._get_default_api_base()
if api_key is None:
raise ValueError("api_key is required for OpenAI realtime calls")
@ -56,30 +113,27 @@ class OpenAIRealtime(OpenAIChatCompletion):
url = self._construct_url(api_base, query_params)
try:
# Only use SSL context for secure websocket connections (wss://)
# websockets library doesn't accept ssl argument for ws:// URIs
ssl_context = None if url.startswith("ws://") else get_shared_realtime_ssl_context()
# Get provider-specific SSL configuration
ssl_config = self._get_ssl_config(url)
# Get provider-specific headers
headers = self._get_additional_headers(api_key)
# Log a masked request preview consistent with other endpoints.
logging_obj.pre_call(
input=None,
api_key=api_key,
additional_args={
"api_base": url,
"headers": {
"Authorization": f"Bearer {api_key}",
"OpenAI-Beta": "realtime=v1",
},
"headers": headers,
"complete_input_dict": {"query_params": query_params},
},
)
async with websockets.connect( # type: ignore
url,
additional_headers={
"Authorization": f"Bearer {api_key}", # type: ignore
"OpenAI-Beta": "realtime=v1",
},
additional_headers=headers, # type: ignore
max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
ssl=ssl_context,
ssl=ssl_config,
) as backend_ws:
realtime_streaming = RealTimeStreaming(
websocket, cast(ClientConnection, backend_ws), logging_obj

51
litellm/llms/openai/responses/guardrail_translation/handler.py
View file

@ -319,9 +319,7 @@ class OpenAIResponsesHandler(BaseTranslation):
return response
if not response_output:
verbose_proxy_logger.debug(
"OpenAI Responses API: Empty output in response"
)
verbose_proxy_logger.debug("OpenAI Responses API: Empty output in response")
return response
# Step 1: Extract all text content and tool calls from response output
@ -427,27 +425,30 @@ class OpenAIResponsesHandler(BaseTranslation):
handle_raw_dict_callback=None,
)
tool_calls = model_response_choices[0].message.tool_calls
text = model_response_choices[0].message.content
guardrail_inputs = GenericGuardrailAPIInputs()
if text:
guardrail_inputs["texts"] = [text]
if tool_calls:
guardrail_inputs["tool_calls"] = cast(
List[ChatCompletionToolCallChunk], tool_calls
)
# Include model information from the response if available
response_model = final_chunk.get("response", {}).get("model")
if response_model:
guardrail_inputs["model"] = response_model
if tool_calls or text:
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=guardrail_inputs,
request_data={},
input_type="response",
logging_obj=litellm_logging_obj,
)
return responses_so_far
if model_response_choices:
tool_calls = model_response_choices[0].message.tool_calls
text = model_response_choices[0].message.content
guardrail_inputs = GenericGuardrailAPIInputs()
if text:
guardrail_inputs["texts"] = [text]
if tool_calls:
guardrail_inputs["tool_calls"] = cast(
List[ChatCompletionToolCallChunk], tool_calls
)
# Include model information from the response if available
response_model = final_chunk.get("response", {}).get("model")
if response_model:
guardrail_inputs["model"] = response_model
if tool_calls or text:
_guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=guardrail_inputs,
request_data={},
input_type="response",
logging_obj=litellm_logging_obj,
)
return responses_so_far
else:
verbose_proxy_logger.debug("Skipping output guardrail - model response has no choices")
# model_response_stream = OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(final_chunk)
# tool_calls = model_response_stream.choices[0].tool_calls
# convert openai response to model response
@ -513,11 +514,9 @@ class OpenAIResponsesHandler(BaseTranslation):
# Check if it's an OutputText with text
if isinstance(content_item, OutputText):
if content_item.text:
return True
elif isinstance(content_item, dict):
if content_item.get("text"):
return True
return False

55
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -1732,6 +1732,52 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
else:
return "stop"
@staticmethod
def _check_prompt_level_content_filter(
processed_chunk: GenerateContentResponseBody,
response_id: Optional[str],
) -> Optional["ModelResponseStream"]:
"""
Check if prompt is blocked due to content filtering at the prompt level.
This handles the case where Vertex AI blocks the prompt before generation begins,
indicated by promptFeedback.blockReason being present.
Args:
processed_chunk: The parsed response chunk from Vertex AI
response_id: The response ID from the chunk
Returns:
ModelResponseStream with content_filter finish_reason if blocked, None otherwise.
Note:
This is consistent with non-streaming _handle_blocked_response() behavior.
Candidate-level content filtering (SAFETY, RECITATION, etc.) is handled
separately via _process_candidates() → _check_finish_reason().
"""
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
# Check if prompt is blocked due to content filtering
prompt_feedback = processed_chunk.get("promptFeedback")
if prompt_feedback and "blockReason" in prompt_feedback:
verbose_logger.debug(
f"Prompt blocked due to: {prompt_feedback.get('blockReason')} - {prompt_feedback.get('blockReasonMessage')}"
)
# Create a content_filter response (consistent with non-streaming _handle_blocked_response)
choice = StreamingChoices(
finish_reason="content_filter",
index=0,
delta=Delta(content=None, role="assistant"),
logprobs=None,
enhancements=None,
)
model_response = ModelResponseStream(choices=[choice], id=response_id)
return model_response
return None
@staticmethod
def _calculate_web_search_requests(grounding_metadata: List[dict]) -> Optional[int]:
web_search_requests: Optional[int] = None
@ -2813,6 +2859,15 @@ class ModelResponseIterator:
processed_chunk = GenerateContentResponseBody(**chunk) # type: ignore
response_id = processed_chunk.get("responseId")
model_response = ModelResponseStream(choices=[], id=response_id)
# Check if prompt is blocked due to content filtering
blocked_response = VertexGeminiConfig._check_prompt_level_content_filter(
processed_chunk=processed_chunk,
response_id=response_id,
)
if blocked_response is not None:
model_response = blocked_response
usage: Optional[Usage] = None
_candidates: Optional[List[Candidates]] = processed_chunk.get("candidates")
grounding_metadata: List[dict] = []

3
litellm/llms/xai/chat/transformation.py
View file

@ -4,6 +4,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.constants import XAI_API_BASE
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
strip_name_from_messages,
@ -14,8 +15,6 @@ from litellm.types.utils import Choices, ModelResponse, Usage, PromptTokensDetai
from ...openai.chat.gpt_transformation import OpenAIGPTConfig
XAI_API_BASE = "https://api.x.ai/v1"
class XAIChatConfig(OpenAIGPTConfig):
@property

5
litellm/llms/xai/realtime/__init__.py Normal file
View file

@ -0,0 +1,5 @@
"""xAI Realtime API handler."""
from .handler import XAIRealtime
__all__ = ["XAIRealtime"]

38
litellm/llms/xai/realtime/handler.py Normal file
View file

@ -0,0 +1,38 @@
"""
This file contains the handler for xAI's Grok Voice Agent API `/v1/realtime` endpoint.
xAI's Realtime API is fully OpenAI-compatible, so we inherit from OpenAIRealtime
and only override the configuration differences.
This requires websockets, and is currently only supported on LiteLLM Proxy.
"""
from litellm.constants import XAI_API_BASE
from ...openai.realtime.handler import OpenAIRealtime
class XAIRealtime(OpenAIRealtime):
"""
Handler for xAI Grok Voice Agent API.
xAI's Realtime API uses the same WebSocket protocol as OpenAI but with:
- Different endpoint: wss://api.x.ai/v1/realtime (via _get_default_api_base)
- No OpenAI-Beta header required (via _get_additional_headers)
- Model: grok-4-1-fast-non-reasoning
All WebSocket logic is inherited from OpenAIRealtime.
"""
def _get_default_api_base(self) -> str:
"""xAI uses a different API base URL."""
return XAI_API_BASE
def _get_additional_headers(self, api_key: str) -> dict:
"""
xAI does NOT require the OpenAI-Beta header.
Only send Authorization header.
"""
return {
"Authorization": f"Bearer {api_key}",
}

3
litellm/llms/xai/responses/transformation.py
View file

@ -2,6 +2,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
import litellm
from litellm._logging import verbose_logger
from litellm.constants import XAI_API_BASE
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
@ -16,8 +17,6 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
XAI_API_BASE = "https://api.x.ai/v1"
class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
"""

50
litellm/main.py
View file

@ -1199,6 +1199,13 @@ def completion( # type: ignore # noqa: PLR0915
headers = {}
if extra_headers is not None:
headers.update(extra_headers)
# Inject proxy auth headers if configured
if litellm.proxy_auth is not None:
try:
proxy_headers = litellm.proxy_auth.get_auth_headers()
headers.update(proxy_headers)
except Exception as e:
verbose_logger.warning(f"Failed to get proxy auth headers: {e}")
num_retries = kwargs.get(
"num_retries", None
) ## alt. param for 'max_retries'. Use this to pass retries w/ instructor.
@ -2201,14 +2208,24 @@ def completion( # type: ignore # noqa: PLR0915
)
elif custom_llm_provider == "a2a":
# A2A (Agent-to-Agent) Protocol
api_base = (
api_base
or litellm.api_base
or get_secret_str("A2A_API_BASE")
# Resolve agent configuration from registry if model format is "a2a/<agent-name>"
api_base, api_key, headers = litellm.A2AConfig.resolve_agent_config_from_registry(
model=model,
api_base=api_base,
api_key=api_key,
headers=headers,
optional_params=optional_params,
)
# Fall back to environment variables and defaults
api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE")
if api_base is None:
raise Exception("api_base is required for A2A provider")
raise Exception(
"api_base is required for A2A provider. "
"Either provide api_base parameter, set A2A_API_BASE environment variable, "
"or register the agent in the proxy with model='a2a/<agent-name>'."
)
headers = headers or litellm.headers
@ -2487,6 +2504,20 @@ def completion( # type: ignore # noqa: PLR0915
headers = headers or litellm.headers
# Add GitHub Copilot headers (same as /responses endpoint does)
if custom_llm_provider == "github_copilot":
from litellm.llms.github_copilot.common_utils import (
get_copilot_default_headers,
)
from litellm.llms.github_copilot.authenticator import Authenticator
copilot_auth = Authenticator()
copilot_api_key = copilot_auth.get_api_key()
copilot_headers = get_copilot_default_headers(copilot_api_key)
if extra_headers:
copilot_headers.update(extra_headers)
extra_headers = copilot_headers
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
@ -4587,6 +4618,13 @@ def embedding( # noqa: PLR0915
headers = {}
if extra_headers is not None:
headers.update(extra_headers)
# Inject proxy auth headers if configured
if litellm.proxy_auth is not None:
try:
proxy_headers = litellm.proxy_auth.get_auth_headers()
headers.update(proxy_headers)
except Exception as e:
verbose_logger.warning(f"Failed to get proxy auth headers: {e}")
### CUSTOM MODEL COST ###
input_cost_per_token = kwargs.get("input_cost_per_token", None)
output_cost_per_token = kwargs.get("output_cost_per_token", None)

469
litellm/model_prices_and_context_window_backup.json
View file

@ -744,7 +744,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 346
"tool_use_system_prompt_tokens": 346,
"supports_native_streaming": true
},
"anthropic.claude-3-5-sonnet-20240620-v1:0": {
"input_cost_per_token": 3e-06,
@ -12850,6 +12851,40 @@
"supports_vision": true,
"supports_web_search": true
},
"deep-research-pro-preview-12-2025": {
"input_cost_per_image": 0.0011,
"input_cost_per_token": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 65536,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "image_generation",
"output_cost_per_image": 0.134,
"output_cost_per_image_token": 0.00012,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_batches": 6e-06,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true
},
"gemini-2.5-flash-lite": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_audio_token": 3e-07,
@ -13304,7 +13339,8 @@
"supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_native_streaming": true
},
"vertex_ai/gemini-3-pro-preview": {
"cache_read_input_token_cost": 2e-07,
@ -13352,7 +13388,8 @@
"supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_native_streaming": true
},
"vertex_ai/gemini-3-flash-preview": {
"cache_read_input_token_cost": 5e-08,
@ -13395,7 +13432,8 @@
"supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_native_streaming": true
},
"gemini-2.5-pro-exp-03-25": {
"cache_read_input_token_cost": 1.25e-07,
@ -14762,6 +14800,42 @@
"supports_vision": true,
"supports_web_search": true
},
"gemini/deep-research-pro-preview-12-2025": {
"input_cost_per_image": 0.0011,
"input_cost_per_token": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "gemini",
"max_input_tokens": 65536,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "image_generation",
"output_cost_per_image": 0.134,
"output_cost_per_image_token": 0.00012,
"output_cost_per_token": 1.2e-05,
"rpm": 1000,
"tpm": 4000000,
"output_cost_per_token_batches": 6e-06,
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true
},
"gemini/gemini-2.5-flash-lite": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_audio_token": 3e-07,
@ -15346,6 +15420,7 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 800000
},
"gemini-3-flash-preview": {
@ -15391,7 +15466,8 @@
"supports_tool_choice": true,
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true
"supports_web_search": true,
"supports_native_streaming": true
},
"gemini/gemini-2.5-pro-exp-03-25": {
"cache_read_input_token_cost": 0.0,
@ -24343,6 +24419,31 @@
"supports_tool_choice": true,
"supports_function_calling": true
},
"openrouter/qwen/qwen3-235b-a22b-2507": {
"input_cost_per_token": 7.1e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 1e-07,
"source": "https://openrouter.ai/qwen/qwen3-235b-a22b-2507",
"supports_function_calling": true,
"supports_tool_choice": true
},
"openrouter/qwen/qwen3-235b-a22b-thinking-2507": {
"input_cost_per_token": 1.1e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://openrouter.ai/qwen/qwen3-235b-a22b-thinking-2507",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true
},
"openrouter/switchpoint/router": {
"input_cost_per_token": 8.5e-07,
"litellm_provider": "openrouter",
@ -27857,7 +27958,9 @@
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_token": 3e-07
"output_cost_per_token": 3e-07,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/alibaba/qwen3-coder": {
"input_cost_per_token": 4e-07,
@ -27866,7 +27969,9 @@
"max_output_tokens": 66536,
"max_tokens": 66536,
"mode": "chat",
"output_cost_per_token": 1.6e-06
"output_cost_per_token": 1.6e-06,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/amazon/nova-lite": {
"input_cost_per_token": 6e-08,
@ -27875,7 +27980,10 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2.4e-07
"output_cost_per_token": 2.4e-07,
"supports_vision": true,
"supports_function_calling": true,
"supports_response_schema": true
},
"vercel_ai_gateway/amazon/nova-micro": {
"input_cost_per_token": 3.5e-08,
@ -27884,7 +27992,9 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.4e-07
"output_cost_per_token": 1.4e-07,
"supports_function_calling": true,
"supports_response_schema": true
},
"vercel_ai_gateway/amazon/nova-pro": {
"input_cost_per_token": 8e-07,
@ -27893,7 +28003,10 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.2e-06
"output_cost_per_token": 3.2e-06,
"supports_vision": true,
"supports_function_calling": true,
"supports_response_schema": true
},
"vercel_ai_gateway/amazon/titan-embed-text-v2": {
"input_cost_per_token": 2e-08,
@ -27913,7 +28026,11 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.25e-06
"output_cost_per_token": 1.25e-06,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/anthropic/claude-3-opus": {
"cache_creation_input_token_cost": 1.875e-05,
@ -27924,7 +28041,11 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 7.5e-05
"output_cost_per_token": 7.5e-05,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/anthropic/claude-3.5-haiku": {
"cache_creation_input_token_cost": 1e-06,
@ -27935,7 +28056,11 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4e-06
"output_cost_per_token": 4e-06,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/anthropic/claude-3.5-sonnet": {
"cache_creation_input_token_cost": 3.75e-06,
@ -27946,7 +28071,11 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.5e-05
"output_cost_per_token": 1.5e-05,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/anthropic/claude-3.7-sonnet": {
"cache_creation_input_token_cost": 3.75e-06,
@ -27957,7 +28086,11 @@
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 1.5e-05
"output_cost_per_token": 1.5e-05,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/anthropic/claude-4-opus": {
"cache_creation_input_token_cost": 1.875e-05,
@ -27968,7 +28101,11 @@
"max_output_tokens": 32000,
"max_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 7.5e-05
"output_cost_per_token": 7.5e-05,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/anthropic/claude-4-sonnet": {
"cache_creation_input_token_cost": 3.75e-06,
@ -27979,7 +28116,9 @@
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 1.5e-05
"output_cost_per_token": 1.5e-05,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/cohere/command-a": {
"input_cost_per_token": 2.5e-06,
@ -27988,7 +28127,9 @@
"max_output_tokens": 8000,
"max_tokens": 8000,
"mode": "chat",
"output_cost_per_token": 1e-05
"output_cost_per_token": 1e-05,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/cohere/command-r": {
"input_cost_per_token": 1.5e-07,
@ -27997,7 +28138,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 6e-07
"output_cost_per_token": 6e-07,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/cohere/command-r-plus": {
"input_cost_per_token": 2.5e-06,
@ -28006,7 +28149,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1e-05
"output_cost_per_token": 1e-05,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/cohere/embed-v4.0": {
"input_cost_per_token": 1.2e-07,
@ -28024,7 +28169,8 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2.19e-06
"output_cost_per_token": 2.19e-06,
"supports_tool_choice": true
},
"vercel_ai_gateway/deepseek/deepseek-r1-distill-llama-70b": {
"input_cost_per_token": 7.5e-07,
@ -28033,7 +28179,10 @@
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 9.9e-07
"output_cost_per_token": 9.9e-07,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/deepseek/deepseek-v3": {
"input_cost_per_token": 9e-07,
@ -28042,7 +28191,8 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 9e-07
"output_cost_per_token": 9e-07,
"supports_tool_choice": true
},
"vercel_ai_gateway/google/gemini-2.0-flash": {
"deprecation_date": "2026-03-31",
@ -28052,7 +28202,11 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 6e-07
"output_cost_per_token": 6e-07,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/google/gemini-2.0-flash-lite": {
"deprecation_date": "2026-03-31",
@ -28062,7 +28216,11 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3e-07
"output_cost_per_token": 3e-07,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/google/gemini-2.5-flash": {
"input_cost_per_token": 3e-07,
@ -28071,7 +28229,11 @@
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 2.5e-06
"output_cost_per_token": 2.5e-06,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/google/gemini-2.5-pro": {
"input_cost_per_token": 2.5e-06,
@ -28080,7 +28242,11 @@
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 1e-05
"output_cost_per_token": 1e-05,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/google/gemini-embedding-001": {
"input_cost_per_token": 1.5e-07,
@ -28098,7 +28264,10 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2e-07
"output_cost_per_token": 2e-07,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/google/text-embedding-005": {
"input_cost_per_token": 2.5e-08,
@ -28134,7 +28303,8 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 7.9e-07
"output_cost_per_token": 7.9e-07,
"supports_tool_choice": true
},
"vercel_ai_gateway/meta/llama-3-8b": {
"input_cost_per_token": 5e-08,
@ -28143,7 +28313,8 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 8e-08
"output_cost_per_token": 8e-08,
"supports_tool_choice": true
},
"vercel_ai_gateway/meta/llama-3.1-70b": {
"input_cost_per_token": 7.2e-07,
@ -28152,7 +28323,8 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 7.2e-07
"output_cost_per_token": 7.2e-07,
"supports_tool_choice": true
},
"vercel_ai_gateway/meta/llama-3.1-8b": {
"input_cost_per_token": 5e-08,
@ -28161,7 +28333,9 @@
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 8e-08
"output_cost_per_token": 8e-08,
"supports_function_calling": true,
"supports_response_schema": true
},
"vercel_ai_gateway/meta/llama-3.2-11b": {
"input_cost_per_token": 1.6e-07,
@ -28170,7 +28344,10 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.6e-07
"output_cost_per_token": 1.6e-07,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/meta/llama-3.2-1b": {
"input_cost_per_token": 1e-07,
@ -28188,7 +28365,9 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 1.5e-07
"output_cost_per_token": 1.5e-07,
"supports_function_calling": true,
"supports_response_schema": true
},
"vercel_ai_gateway/meta/llama-3.2-90b": {
"input_cost_per_token": 7.2e-07,
@ -28197,7 +28376,10 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 7.2e-07
"output_cost_per_token": 7.2e-07,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/meta/llama-3.3-70b": {
"input_cost_per_token": 7.2e-07,
@ -28206,7 +28388,9 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 7.2e-07
"output_cost_per_token": 7.2e-07,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/meta/llama-4-maverick": {
"input_cost_per_token": 2e-07,
@ -28215,7 +28399,8 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 6e-07
"output_cost_per_token": 6e-07,
"supports_tool_choice": true
},
"vercel_ai_gateway/meta/llama-4-scout": {
"input_cost_per_token": 1e-07,
@ -28224,7 +28409,10 @@
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3e-07
"output_cost_per_token": 3e-07,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/mistral/codestral": {
"input_cost_per_token": 3e-07,
@ -28233,7 +28421,9 @@
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 9e-07
"output_cost_per_token": 9e-07,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/mistral/codestral-embed": {
"input_cost_per_token": 1.5e-07,
@ -28251,7 +28441,10 @@
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 2.8e-07
"output_cost_per_token": 2.8e-07,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/mistral/magistral-medium": {
"input_cost_per_token": 2e-06,
@ -28260,7 +28453,10 @@
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 5e-06
"output_cost_per_token": 5e-06,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/mistral/magistral-small": {
"input_cost_per_token": 5e-07,
@ -28269,7 +28465,8 @@
"max_output_tokens": 64000,
"max_tokens": 64000,
"mode": "chat",
"output_cost_per_token": 1.5e-06
"output_cost_per_token": 1.5e-06,
"supports_function_calling": true
},
"vercel_ai_gateway/mistral/ministral-3b": {
"input_cost_per_token": 4e-08,
@ -28278,7 +28475,9 @@
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 4e-08
"output_cost_per_token": 4e-08,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/mistral/ministral-8b": {
"input_cost_per_token": 1e-07,
@ -28287,7 +28486,10 @@
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 1e-07
"output_cost_per_token": 1e-07,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/mistral/mistral-embed": {
"input_cost_per_token": 1e-07,
@ -28305,7 +28507,9 @@
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 6e-06
"output_cost_per_token": 6e-06,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/mistral/mistral-saba-24b": {
"input_cost_per_token": 7.9e-07,
@ -28323,7 +28527,10 @@
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 3e-07
"output_cost_per_token": 3e-07,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/mistral/mixtral-8x22b-instruct": {
"input_cost_per_token": 1.2e-06,
@ -28332,7 +28539,8 @@
"max_output_tokens": 2048,
"max_tokens": 2048,
"mode": "chat",
"output_cost_per_token": 1.2e-06
"output_cost_per_token": 1.2e-06,
"supports_function_calling": true
},
"vercel_ai_gateway/mistral/pixtral-12b": {
"input_cost_per_token": 1.5e-07,
@ -28341,7 +28549,11 @@
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 1.5e-07
"output_cost_per_token": 1.5e-07,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/mistral/pixtral-large": {
"input_cost_per_token": 2e-06,
@ -28350,7 +28562,11 @@
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 6e-06
"output_cost_per_token": 6e-06,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/moonshotai/kimi-k2": {
"input_cost_per_token": 5.5e-07,
@ -28359,7 +28575,9 @@
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_token": 2.2e-06
"output_cost_per_token": 2.2e-06,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/morph/morph-v3-fast": {
"input_cost_per_token": 8e-07,
@ -28386,7 +28604,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 1.5e-06
"output_cost_per_token": 1.5e-06,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/openai/gpt-3.5-turbo-instruct": {
"input_cost_per_token": 1.5e-06,
@ -28404,7 +28624,10 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "chat",
"output_cost_per_token": 3e-05
"output_cost_per_token": 3e-05,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/openai/gpt-4.1": {
"cache_creation_input_token_cost": 0.0,
@ -28415,7 +28638,11 @@
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 8e-06
"output_cost_per_token": 8e-06,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/openai/gpt-4.1-mini": {
"cache_creation_input_token_cost": 0.0,
@ -28426,7 +28653,11 @@
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 1.6e-06
"output_cost_per_token": 1.6e-06,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/openai/gpt-4.1-nano": {
"cache_creation_input_token_cost": 0.0,
@ -28437,7 +28668,11 @@
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 4e-07
"output_cost_per_token": 4e-07,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/openai/gpt-4o": {
"cache_creation_input_token_cost": 0.0,
@ -28448,7 +28683,11 @@
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_token": 1e-05
"output_cost_per_token": 1e-05,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/openai/gpt-4o-mini": {
"cache_creation_input_token_cost": 0.0,
@ -28459,7 +28698,11 @@
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_token": 6e-07
"output_cost_per_token": 6e-07,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/openai/o1": {
"cache_creation_input_token_cost": 0.0,
@ -28470,7 +28713,11 @@
"max_output_tokens": 100000,
"max_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 6e-05
"output_cost_per_token": 6e-05,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/openai/o3": {
"cache_creation_input_token_cost": 0.0,
@ -28481,7 +28728,11 @@
"max_output_tokens": 100000,
"max_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 8e-06
"output_cost_per_token": 8e-06,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/openai/o3-mini": {
"cache_creation_input_token_cost": 0.0,
@ -28492,7 +28743,10 @@
"max_output_tokens": 100000,
"max_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 4.4e-06
"output_cost_per_token": 4.4e-06,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/openai/o4-mini": {
"cache_creation_input_token_cost": 0.0,
@ -28503,7 +28757,11 @@
"max_output_tokens": 100000,
"max_tokens": 100000,
"mode": "chat",
"output_cost_per_token": 4.4e-06
"output_cost_per_token": 4.4e-06,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_response_schema": true
},
"vercel_ai_gateway/openai/text-embedding-3-large": {
"input_cost_per_token": 1.3e-07,
@ -28575,7 +28833,10 @@
"max_output_tokens": 32000,
"max_tokens": 32000,
"mode": "chat",
"output_cost_per_token": 1.5e-05
"output_cost_per_token": 1.5e-05,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/vercel/v0-1.5-md": {
"input_cost_per_token": 3e-06,
@ -28584,7 +28845,10 @@
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 1.5e-05
"output_cost_per_token": 1.5e-05,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/xai/grok-2": {
"input_cost_per_token": 2e-06,
@ -28593,7 +28857,9 @@
"max_output_tokens": 4000,
"max_tokens": 4000,
"mode": "chat",
"output_cost_per_token": 1e-05
"output_cost_per_token": 1e-05,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/xai/grok-2-vision": {
"input_cost_per_token": 2e-06,
@ -28602,7 +28868,10 @@
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "chat",
"output_cost_per_token": 1e-05
"output_cost_per_token": 1e-05,
"supports_vision": true,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/xai/grok-3": {
"input_cost_per_token": 3e-06,
@ -28611,7 +28880,9 @@
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.5e-05
"output_cost_per_token": 1.5e-05,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/xai/grok-3-fast": {
"input_cost_per_token": 5e-06,
@ -28620,7 +28891,8 @@
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2.5e-05
"output_cost_per_token": 2.5e-05,
"supports_function_calling": true
},
"vercel_ai_gateway/xai/grok-3-mini": {
"input_cost_per_token": 3e-07,
@ -28629,7 +28901,9 @@
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 5e-07
"output_cost_per_token": 5e-07,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/xai/grok-3-mini-fast": {
"input_cost_per_token": 6e-07,
@ -28638,7 +28912,9 @@
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4e-06
"output_cost_per_token": 4e-06,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/xai/grok-4": {
"input_cost_per_token": 3e-06,
@ -28647,7 +28923,9 @@
"max_output_tokens": 256000,
"max_tokens": 256000,
"mode": "chat",
"output_cost_per_token": 1.5e-05
"output_cost_per_token": 1.5e-05,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/zai/glm-4.5": {
"input_cost_per_token": 6e-07,
@ -28656,7 +28934,9 @@
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2.2e-06
"output_cost_per_token": 2.2e-06,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/zai/glm-4.5-air": {
"input_cost_per_token": 2e-07,
@ -28665,7 +28945,9 @@
"max_output_tokens": 96000,
"max_tokens": 96000,
"mode": "chat",
"output_cost_per_token": 1.1e-06
"output_cost_per_token": 1.1e-06,
"supports_function_calling": true,
"supports_tool_choice": true
},
"vercel_ai_gateway/zai/glm-4.6": {
"litellm_provider": "vercel_ai_gateway",
@ -28733,7 +29015,9 @@
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
"supports_tool_choice": true,
"supports_native_streaming": true,
"supports_vision": true
},
"vertex_ai/claude-3-5-sonnet": {
"input_cost_per_token": 3e-06,
@ -29004,7 +29288,8 @@
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tool_use_system_prompt_tokens": 159
"tool_use_system_prompt_tokens": 159,
"supports_native_streaming": true
},
"vertex_ai/claude-sonnet-4-5": {
"cache_creation_input_token_cost": 3.75e-06,
@ -29056,7 +29341,8 @@
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
"supports_vision": true,
"supports_native_streaming": true
},
"vertex_ai/claude-opus-4@20250514": {
"cache_creation_input_token_cost": 1.875e-05,
@ -29338,6 +29624,21 @@
"output_cost_per_token_batches": 6e-06,
"source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image"
},
"vertex_ai/deep-research-pro-preview-12-2025": {
"input_cost_per_image": 0.0011,
"input_cost_per_token": 2e-06,
"input_cost_per_token_batches": 1e-06,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 65536,
"max_output_tokens": 32768,
"max_tokens": 32768,
"mode": "image_generation",
"output_cost_per_image": 0.134,
"output_cost_per_image_token": 0.00012,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_batches": 6e-06,
"source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image"
},
"vertex_ai/imagegeneration@006": {
"litellm_provider": "vertex_ai-image-models",
"mode": "image_generation",
@ -29827,7 +30128,9 @@
"mode": "chat",
"output_cost_per_token": 1e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_regions": ["global"],
"supported_regions": [
"global"
],
"supports_function_calling": true,
"supports_tool_choice": true
},
@ -29840,7 +30143,9 @@
"mode": "chat",
"output_cost_per_token": 4e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_regions": ["global"],
"supported_regions": [
"global"
],
"supports_function_calling": true,
"supports_tool_choice": true
},
@ -29853,7 +30158,9 @@
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_regions": ["global"],
"supported_regions": [
"global"
],
"supports_function_calling": true,
"supports_tool_choice": true
},
@ -29866,7 +30173,9 @@
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_regions": ["global"],
"supported_regions": [
"global"
],
"supports_function_calling": true,
"supports_tool_choice": true
},
@ -34815,4 +35124,4 @@
"output_cost_per_token": 0,
"supports_reasoning": true
}
}
}

3
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -387,6 +387,9 @@ class MCPRequestHandler:
user_api_key_cache,
)
verbose_logger.debug(
f"MCP team permission lookup: team_id={user_api_key_auth.team_id if user_api_key_auth else None}"
)
if not user_api_key_auth or not user_api_key_auth.team_id or not prisma_client:
return None

0
litellm/proxy/_experimental/out/404.html → litellm/proxy/_experimental/out/404/index.html
View file

0
litellm/proxy/_experimental/out/api-reference.html → litellm/proxy/_experimental/out/api-reference/index.html
View file

0
litellm/proxy/_experimental/out/experimental/api-playground.html → litellm/proxy/_experimental/out/experimental/api-playground/index.html
View file

0
litellm/proxy/_experimental/out/experimental/budgets.html → litellm/proxy/_experimental/out/experimental/budgets/index.html
View file

0
litellm/proxy/_experimental/out/experimental/caching.html → litellm/proxy/_experimental/out/experimental/caching/index.html
View file

0
litellm/proxy/_experimental/out/experimental/claude-code-plugins.html → litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html
View file

0
litellm/proxy/_experimental/out/experimental/old-usage.html → litellm/proxy/_experimental/out/experimental/old-usage/index.html
View file

0
litellm/proxy/_experimental/out/experimental/prompts.html → litellm/proxy/_experimental/out/experimental/prompts/index.html
View file

0
litellm/proxy/_experimental/out/experimental/tag-management.html → litellm/proxy/_experimental/out/experimental/tag-management/index.html
View file

0
litellm/proxy/_experimental/out/guardrails.html → litellm/proxy/_experimental/out/guardrails/index.html
View file

0
litellm/proxy/_experimental/out/login.html → litellm/proxy/_experimental/out/login/index.html
View file

0
litellm/proxy/_experimental/out/logs.html → litellm/proxy/_experimental/out/logs/index.html
View file

0
litellm/proxy/_experimental/out/mcp/oauth/callback.html → litellm/proxy/_experimental/out/mcp/oauth/callback/index.html
View file

0
litellm/proxy/_experimental/out/model-hub.html → litellm/proxy/_experimental/out/model-hub/index.html
View file

0
litellm/proxy/_experimental/out/model_hub.html → litellm/proxy/_experimental/out/model_hub/index.html
View file

0
litellm/proxy/_experimental/out/model_hub_table.html → litellm/proxy/_experimental/out/model_hub_table/index.html
View file

0
litellm/proxy/_experimental/out/models-and-endpoints.html → litellm/proxy/_experimental/out/models-and-endpoints/index.html
View file

0
litellm/proxy/_experimental/out/onboarding.html → litellm/proxy/_experimental/out/onboarding/index.html
View file

0
litellm/proxy/_experimental/out/organizations.html → litellm/proxy/_experimental/out/organizations/index.html
View file

0
litellm/proxy/_experimental/out/playground.html → litellm/proxy/_experimental/out/playground/index.html
View file

0
litellm/proxy/_experimental/out/policies.html → litellm/proxy/_experimental/out/policies/index.html
View file

0
litellm/proxy/_experimental/out/settings/admin-settings.html → litellm/proxy/_experimental/out/settings/admin-settings/index.html
View file

0
litellm/proxy/_experimental/out/settings/logging-and-alerts.html → litellm/proxy/_experimental/out/settings/logging-and-alerts/index.html
View file

0
litellm/proxy/_experimental/out/settings/router-settings.html → litellm/proxy/_experimental/out/settings/router-settings/index.html
View file

0
litellm/proxy/_experimental/out/settings/ui-theme.html → litellm/proxy/_experimental/out/settings/ui-theme/index.html
View file

0
litellm/proxy/_experimental/out/teams.html → litellm/proxy/_experimental/out/teams/index.html
View file

0
litellm/proxy/_experimental/out/test-key.html → litellm/proxy/_experimental/out/test-key/index.html
View file

0
litellm/proxy/_experimental/out/tools/mcp-servers.html → litellm/proxy/_experimental/out/tools/mcp-servers/index.html
View file

0
litellm/proxy/_experimental/out/tools/vector-stores.html → litellm/proxy/_experimental/out/tools/vector-stores/index.html
View file

0
litellm/proxy/_experimental/out/usage.html → litellm/proxy/_experimental/out/usage/index.html
View file

Some files were not shown because too many files have changed in this diff Show more