scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 87

Merge branch 'litellm_internal_staging' into litellm_vertex_request_metadata_labels
Some checks failed
Unit Tests: Proxy DB Operations / proxy-db (key-generation, tests/proxy_unit_tests/test_key_generate_prisma.py, 30, 0) (push) Has been cancelled
Details
Unit Tests: Proxy DB Operations / proxy-db (auth-checks, tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py, 20, 8) (push) Has been cancelled
Details
Unit Tests: Proxy DB Operations / proxy-db (remaining, tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py, 30, 8) (push) Has been cancelled
Details
Unit Tests: Security / security (push) Has been cancelled
Details

Browse source
This commit is contained in:
Sameer Kankute 2026-04-21 09:10:51 +05:30 • committed by GitHub
commit 2b028a62d8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2097 changed files with 68356 additions and 43718 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
.circleci/config.yml
View file

@ -439,7 +439,14 @@ jobs:
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
- image: cimg/postgres:16.0
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
@ -463,12 +470,14 @@ jobs:
paths:
- ./.venv
key: v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }}
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Run prisma ./docker/entrypoint.sh
name: Seed DB schema via prisma db push
command: |
set +e
chmod +x docker/entrypoint.sh
./docker/entrypoint.sh
uv run --no-sync litellm --skip_server_startup --use_prisma_db_push
set -e
- run:
name: Generate Prisma Client
@ -1707,7 +1716,7 @@ jobs:
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda create -n myenv python=3.10 -y
conda activate myenv
python --version
- run:
@ -1822,7 +1831,7 @@ jobs:
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda create -n myenv python=3.10 -y
conda activate myenv
python --version
- run:
@ -2057,7 +2066,7 @@ jobs:
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda create -n myenv python=3.10 -y
conda activate myenv
python --version
- run:
@ -2208,7 +2217,7 @@ jobs:
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda create -n myenv python=3.10 -y
conda activate myenv
python --version
- run:
@ -2319,7 +2328,7 @@ jobs:
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda create -n myenv python=3.10 -y
conda activate myenv
python --version
- run:
@ -2444,7 +2453,7 @@ jobs:
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda create -n myenv python=3.10 -y
conda activate myenv
python --version
- run:

40
.github/scripts/close_duplicate_issues.py vendored
View file

@ -42,7 +42,9 @@ def gh(*args: str) -> str:
def fetch_open_issues(repo: str | None) -> list[dict]:
"""Fetch all open issues (excluding PRs) via gh api --paginate."""
if repo:
endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
endpoint = (
f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
)
else:
endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
cmd = ["api", "--paginate", endpoint]
@ -71,7 +73,9 @@ def close_as_duplicate(
repo_args = ["--repo", repo] if repo else []
if dry_run:
print(f" [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
print(
f" [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}"
)
return
# Add comment
@ -115,7 +119,9 @@ def find_duplicate(
return None
def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
def scan_all(
issues: list[dict], threshold: float, repo: str | None, dry_run: bool
) -> int:
"""Compare every issue against all older issues. Returns count of duplicates found."""
# Sort oldest first
issues.sort(key=lambda i: i["number"])
@ -144,7 +150,11 @@ def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bo
def check_single(
issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
issue_number: int,
issues: list[dict],
threshold: float,
repo: str | None,
dry_run: bool,
) -> bool:
"""Check a single issue against all older open issues. Returns True if duplicate found."""
target = None
@ -178,13 +188,23 @@ def check_single(
def main() -> None:
parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
parser = argparse.ArgumentParser(
description="Detect and close duplicate GitHub issues"
)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--scan", action="store_true", help="Scan all open issues")
mode.add_argument("--issue-number", type=int, help="Check a single issue number")
parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
parser.add_argument(
"--threshold", type=float, default=0.85, help="Similarity threshold (0-1)"
)
parser.add_argument(
"--close",
action="store_true",
help="Actually close duplicates (default is dry-run)",
)
parser.add_argument(
"--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted."
)
args = parser.parse_args()
dry_run = not args.close
@ -200,7 +220,9 @@ def main() -> None:
count = scan_all(issues, args.threshold, args.repo, dry_run)
print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
else:
found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
found = check_single(
args.issue_number, issues, args.threshold, args.repo, dry_run
)
sys.exit(0 if found else 0) # Always exit 0; finding no dup is not an error

20
.github/scripts/scan_keywords.py vendored
View file

@ -67,14 +67,13 @@ def send_webhook(webhook_url: str, payload: dict) -> None:
def _excerpt(text: str, max_len: int = 400) -> str:
if not text:
return ""
# Keep original formatting
if len(text) <= max_len:
return text
return text[: max_len - 1] + "…"
def main() -> int:
event = read_event_payload()
if not event:
@ -87,8 +86,19 @@ def main() -> int:
# Keywords from env or defaults
keywords_env = os.environ.get("KEYWORDS", "")
default_keywords = ["azure", "openai", "bedrock", "vertexai", "vertex ai", "anthropic"]
keywords = [k.strip() for k in keywords_env.split(",")] if keywords_env else default_keywords
default_keywords = [
"azure",
"openai",
"bedrock",
"vertexai",
"vertex ai",
"anthropic",
]
keywords = (
[k.strip() for k in keywords_env.split(",")]
if keywords_env
else default_keywords
)
matches = detect_keywords(combined_text, keywords)
found = bool(matches)
@ -129,5 +139,3 @@ def main() -> int:
if __name__ == "__main__":
raise SystemExit(main())

2
CLAUDE.md
View file

@ -110,7 +110,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
### UI Component Library
- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, plain `<span>`/`<div>` with Tailwind classes (or `Typography.Text`) for text, `Card` from `antd`, etc. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, `Typography.Text` / `Typography.Title` / `Typography.Paragraph` for textual content (avoid plain text-only `<span>`, `<p>`, `<h*>` when Typography fits), and `Card` from `antd`. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
### MCP OAuth / OpenAPI Transport Mapping
- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).

4
ci_cd/run_migration.py
View file

@ -17,7 +17,9 @@ def create_migration(migration_name: str = None):
try:
# Get paths
root_dir = Path(__file__).parent.parent
migrations_dir = root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
migrations_dir = (
root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
)
schema_path = root_dir / "schema.prisma"
# Create temporary PostgreSQL database

50
cookbook/anthropic_agent_sdk/agent_with_mcp.py
View file

@ -24,24 +24,26 @@ async def interactive_chat_with_mcp():
Interactive CLI chat with the agent and MCP server
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
litellm_base_url = setup_litellm_env(config)
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
available_models = await fetch_available_models(
litellm_base_url, config.LITELLM_API_KEY
)
current_model = config.LITELLM_MODEL
# MCP server configuration
mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
if not use_mcp:
print("⚠️ MCP disabled via USE_MCP=false")
print_header(litellm_base_url, current_model, has_mcp=use_mcp)
while True:
# Configure agent options
if use_mcp:
@ -58,7 +60,7 @@ async def interactive_chat_with_mcp():
"url": mcp_server_url,
"headers": {
"Authorization": f"Bearer {config.LITELLM_API_KEY}"
}
},
}
},
)
@ -78,12 +80,12 @@ async def interactive_chat_with_mcp():
model=current_model,
max_turns=50,
)
# Create agent client
try:
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
@ -91,34 +93,36 @@ async def interactive_chat_with_mcp():
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
if user_input.lower() in ["quit", "exit"]:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
if user_input.lower() == "clear":
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
if user_input.lower() == "models":
handle_model_list(available_models, current_model)
continue
if user_input.lower() == 'model':
new_model, should_restart = handle_model_switch(available_models, current_model)
if user_input.lower() == "model":
new_model, should_restart = handle_model_switch(
available_models, current_model
)
if should_restart:
current_model = new_model
conversation_active = False
continue
if not user_input:
continue
# Stream response from agent
await stream_response(client, user_input)
except Exception as e:
print(f"\n❌ Error creating agent client: {e}")
print("This might be an MCP configuration issue. Try running without MCP:")

62
cookbook/anthropic_agent_sdk/common.py
View file

@ -8,13 +8,13 @@ import httpx
class Config:
"""Configuration for LiteLLM Gateway connection"""
# LiteLLM proxy URL (default to local instance)
LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
# LiteLLM API key (master key or virtual key)
LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
# Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
@ -28,7 +28,7 @@ async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
response = await client.get(
f"{base_url}/models",
headers={"Authorization": f"Bearer {api_key}"},
timeout=10.0
timeout=10.0,
)
response.raise_for_status()
data = response.json()
@ -50,7 +50,7 @@ def setup_litellm_env(config: Config):
"""
Configure environment variables to point Agent SDK to LiteLLM
"""
litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
litellm_base_url = config.LITELLM_PROXY_URL.rstrip("/")
os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
return litellm_base_url
@ -87,10 +87,12 @@ def handle_model_list(available_models: list[str], current_model: str):
print(f" {marker} {i}. {model}")
def handle_model_switch(available_models: list[str], current_model: str) -> tuple[str, bool]:
def handle_model_switch(
available_models: list[str], current_model: str
) -> tuple[str, bool]:
"""
Handle model switching
Returns:
tuple: (new_model, should_restart_conversation)
"""
@ -98,7 +100,7 @@ def handle_model_switch(available_models: list[str], current_model: str) -> tupl
for i, model in enumerate(available_models, 1):
marker = "✓" if model == current_model else " "
print(f" {marker} {i}. {model}")
try:
choice = input("\nEnter number (or press Enter to cancel): ").strip()
if choice:
@ -112,7 +114,7 @@ def handle_model_switch(available_models: list[str], current_model: str) -> tupl
print("❌ Invalid choice")
except (ValueError, IndexError):
print("❌ Invalid input")
return current_model, False
@ -120,41 +122,43 @@ async def stream_response(client, user_input: str):
"""
Stream response from the agent
"""
print("\n🤖 Assistant: ", end='', flush=True)
print("\n🤖 Assistant: ", end="", flush=True)
try:
await client.query(user_input)
# Show loading indicator
print("⏳ thinking...", end='', flush=True)
print("⏳ thinking...", end="", flush=True)
# Stream the response
first_chunk = True
async for msg in client.receive_response():
# Clear loading indicator on first message
if first_chunk:
print("\r🤖 Assistant: ", end='', flush=True)
print("\r🤖 Assistant: ", end="", flush=True)
first_chunk = False
# Handle different message types
if hasattr(msg, 'type'):
if msg.type == 'content_block_delta':
if hasattr(msg, "type"):
if msg.type == "content_block_delta":
# Streaming text delta
if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
print(msg.delta.text, end='', flush=True)
elif msg.type == 'content_block_start':
if hasattr(msg, "delta") and hasattr(msg.delta, "text"):
print(msg.delta.text, end="", flush=True)
elif msg.type == "content_block_start":
# Start of content block
if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
print(msg.content_block.text, end='', flush=True)
if hasattr(msg, "content_block") and hasattr(
msg.content_block, "text"
):
print(msg.content_block.text, end="", flush=True)
# Fallback to original content handling
if hasattr(msg, 'content'):
if hasattr(msg, "content"):
for content_block in msg.content:
if hasattr(content_block, 'text'):
print(content_block.text, end='', flush=True)
if hasattr(content_block, "text"):
print(content_block.text, end="", flush=True)
print() # New line after response
except Exception as e:
print(f"\r\n❌ Error: {e}")
print("Please check your LiteLLM gateway is running and configured correctly.")

42
cookbook/anthropic_agent_sdk/main.py
View file

@ -24,17 +24,19 @@ async def interactive_chat():
Interactive CLI chat with the agent
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
litellm_base_url = setup_litellm_env(config)
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
available_models = await fetch_available_models(
litellm_base_url, config.LITELLM_API_KEY
)
current_model = config.LITELLM_MODEL
print_header(litellm_base_url, current_model)
while True:
# Configure agent options for each conversation
options = ClaudeAgentOptions(
@ -42,11 +44,11 @@ async def interactive_chat():
model=current_model,
max_turns=50,
)
# Create agent client
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
@ -54,31 +56,33 @@ async def interactive_chat():
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
if user_input.lower() in ["quit", "exit"]:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
if user_input.lower() == "clear":
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
if user_input.lower() == "models":
handle_model_list(available_models, current_model)
continue
if user_input.lower() == 'model':
new_model, should_restart = handle_model_switch(available_models, current_model)
if user_input.lower() == "model":
new_model, should_restart = handle_model_switch(
available_models, current_model
)
if should_restart:
current_model = new_model
conversation_active = False
continue
if not user_input:
continue
# Stream response from agent
await stream_response(client, user_input)

6
cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py
View file

@ -11,15 +11,15 @@ BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
batch_input_file = client.files.create(
file=open("./bedrock_batch_completions.jsonl", "rb"),
purpose="batch",
extra_body={"target_model_names": BEDROCK_BATCH_MODEL}
extra_body={"target_model_names": BEDROCK_BATCH_MODEL},
)
print(batch_input_file)
# Create batch
batch = client.batches.create(
batch = client.batches.create(
input_file_id=batch_input_file.id,
endpoint="/v1/chat/completions",
completion_window="24h",
metadata={"description": "Test batch job"},
)
print(batch)
print(batch)

19
cookbook/litellm_proxy_server/cli_token_usage.py
View file

@ -8,6 +8,7 @@ in your Python scripts after running `litellm-proxy login`.
from textwrap import indent
import litellm
LITELLM_BASE_URL = "http://localhost:4000/"
@ -15,38 +16,38 @@ def main():
"""Using CLI token with LiteLLM SDK"""
print("🚀 Using CLI Token with LiteLLM SDK")
print("=" * 40)
#litellm._turn_on_debug()
# litellm._turn_on_debug()
# Get the CLI token
api_key = litellm.get_litellm_gateway_api_key()
if not api_key:
print("❌ No CLI token found. Please run 'litellm-proxy login' first.")
return
print("✅ Found CLI token.")
available_models = litellm.get_valid_models(
check_provider_endpoint=True,
custom_llm_provider="litellm_proxy",
api_key=api_key,
api_base=LITELLM_BASE_URL
api_base=LITELLM_BASE_URL,
)
print("✅ Available models:")
if available_models:
for i, model in enumerate(available_models, 1):
print(f" {i:2d}. {model}")
else:
print(" No models available")
# Use with LiteLLM
try:
response = litellm.completion(
model="litellm_proxy/gemini/gemini-2.5-flash",
messages=[{"role": "user", "content": "Hello from CLI token!"}],
api_key=api_key,
base_url=LITELLM_BASE_URL
base_url=LITELLM_BASE_URL,
)
print(f"✅ LLM Response: {response.model_dump_json(indent=4)}")
except Exception as e:
@ -55,7 +56,7 @@ def main():
if __name__ == "__main__":
main()
print("\n💡 Tips:")
print("1. Run 'litellm-proxy login' to authenticate first")
print("2. Replace 'https://your-proxy.com' with your actual proxy URL")

11
cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py
View file

@ -3,11 +3,12 @@ Use LiteLLM Proxy MCP Gateway to call MCP tools.
When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
"""
import openai
client = openai.OpenAI(
api_key="sk-1234", # paste your litellm proxy api key here
base_url="http://localhost:4000" # paste your litellm proxy base url here
api_key="sk-1234", # paste your litellm proxy api key here
base_url="http://localhost:4000", # paste your litellm proxy base url here
)
print("Making API request to Responses API with MCP tools")
@ -17,7 +18,7 @@ response = client.responses.create(
{
"role": "user",
"content": "give me TLDR of what BerriAI/litellm repo is about",
"type": "message"
"type": "message",
}
],
tools=[
@ -25,11 +26,11 @@ response = client.responses.create(
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never"
"require_approval": "never",
}
],
stream=True,
tool_choice="required"
tool_choice="required",
)
for chunk in response:

7
cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py
View file

@ -40,8 +40,10 @@ class InMemorySecretManager(CustomSecretManager):
) -> Optional[str]:
"""Read secret synchronously"""
from litellm._logging import verbose_proxy_logger
verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}")
verbose_proxy_logger.info(
f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}"
)
value = self.secrets.get(secret_name)
verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: READ SECRET: {value}")
return value
@ -76,4 +78,3 @@ class InMemorySecretManager(CustomSecretManager):
del self.secrets[secret_name]
return {"status": "deleted", "secret_name": secret_name}
return {"status": "not_found", "secret_name": secret_name}

75
cookbook/livekit_agent_sdk/main.py
View file

@ -5,6 +5,7 @@ This example shows how to use LiveKit's xAI realtime plugin through LiteLLM prox
LiteLLM acts as a unified interface, allowing you to switch between xAI, OpenAI,
and Azure realtime APIs without changing your agent code.
"""
import asyncio
import json
import os
@ -23,71 +24,79 @@ async def run_voice_agent():
2. Sends a user message
3. Streams back the response
"""
url = f"ws://{PROXY_URL.replace('http://', '').replace('https://', '')}/v1/realtime?model={MODEL}"
headers = {"Authorization": f"Bearer {API_KEY}"}
print(f"🎙️ Connecting to voice agent...")
print(f" Model: {MODEL}")
print(f" Proxy: {PROXY_URL}")
print()
async with websockets.connect(url, additional_headers=headers) as ws:
# Receive initial connection event
initial = json.loads(await ws.recv())
print(f"✅ Connected! Event: {initial['type']}\n")
# Get user input
user_message = input("💬 Your message: ").strip()
if not user_message:
user_message = "Tell me a fun fact about AI!"
print(f"\n🤖 Sending to {MODEL}...\n")
# Send user message
await ws.send(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": user_message}]
}
}))
await ws.send(
json.dumps(
{
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": user_message}],
},
}
)
)
# Request response
await ws.send(json.dumps({
"type": "response.create",
"response": {"modalities": ["text", "audio"]}
}))
await ws.send(
json.dumps(
{
"type": "response.create",
"response": {"modalities": ["text", "audio"]},
}
)
)
# Stream response
print("🎤 Response: ", end='', flush=True)
print("🎤 Response: ", end="", flush=True)
transcript = []
try:
while True:
msg = await asyncio.wait_for(ws.recv(), timeout=15.0)
event = json.loads(msg)
# Capture transcript deltas
if event['type'] == 'response.output_audio_transcript.delta':
delta = event.get('delta', '')
if event["type"] == "response.output_audio_transcript.delta":
delta = event.get("delta", "")
if delta:
print(delta, end='', flush=True)
print(delta, end="", flush=True)
transcript.append(delta)
# Done when response completes
elif event['type'] == 'response.done':
elif event["type"] == "response.done":
break
except asyncio.TimeoutError:
pass
print("\n")
if transcript:
print(f"✅ Complete response: {''.join(transcript)}")
await ws.close()
@ -97,7 +106,7 @@ def main():
print("LiveKit xAI Voice Agent via LiteLLM Proxy")
print("=" * 70)
print()
try:
asyncio.run(run_voice_agent())
except KeyboardInterrupt:

14
cookbook/misc/test_responses_api.py
View file

@ -1,10 +1,9 @@
import base64
from openai import OpenAI
import time
client = OpenAI(
base_url="http://0.0.0.0:4001",
api_key="sk-1234"
)
client = OpenAI(base_url="http://0.0.0.0:4001", api_key="sk-1234")
# Function to encode the image
def encode_image(image_path):
@ -25,7 +24,7 @@ response = client.responses.create(
{
"role": "user",
"content": [
{ "type": "input_text", "text": "what color is the image"},
{"type": "input_text", "text": "what color is the image"},
{
"type": "input_image",
"image_url": f"data:image/jpeg;base64,{base64_image}",
@ -36,7 +35,6 @@ response = client.responses.create(
)
print(response.output_text)
print("response1 id===", response.id)
print("sleeping for 20 seconds...")
@ -45,9 +43,7 @@ print("making follow up request for existing id")
response2 = client.responses.create(
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
previous_response_id=response.id,
input="ok, and what objects are in the image?"
input="ok, and what objects are in the image?",
)
print(response2.output_text)

11
cookbook/nova_sonic_realtime.py
View file

@ -52,11 +52,11 @@ class RealtimeClient:
async def connect(self):
"""Connect to LiteLLM proxy realtime endpoint."""
print(f"Connecting to {self.url}...")
headers = {}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
self.ws = await websockets.connect(
self.url,
additional_headers=headers,
@ -175,7 +175,9 @@ class RealtimeClient:
try:
while self.is_active:
audio_data = self.input_stream.read(CHUNK_SIZE, exception_on_overflow=False)
audio_data = self.input_stream.read(
CHUNK_SIZE, exception_on_overflow=False
)
await self.send_audio_chunk(audio_data)
await asyncio.sleep(0.01) # Small delay to prevent overwhelming
except Exception as e:
@ -270,6 +272,7 @@ async def main():
except Exception as e:
print(f"\n❌ Error: {e}")
import traceback
traceback.print_exc()
finally:
await client.close()
@ -281,7 +284,7 @@ if __name__ == "__main__":
print("2. Bedrock is configured in proxy_server_config.yaml")
print("3. AWS credentials are set")
print()
try:
asyncio.run(main())
except KeyboardInterrupt:

164
cookbook/veo_video_generation.py
View file

@ -21,49 +21,45 @@ from typing import Optional
class VeoVideoGenerator:
"""Complete Veo video generation client using LiteLLM proxy."""
def __init__(self, base_url: str = "http://localhost:4000/gemini/v1beta",
api_key: str = "sk-1234"):
def __init__(
self,
base_url: str = "http://localhost:4000/gemini/v1beta",
api_key: str = "sk-1234",
):
"""
Initialize the Veo video generator.
Args:
base_url: Base URL for the LiteLLM proxy with Gemini pass-through
api_key: API key for LiteLLM proxy authentication
"""
self.base_url = base_url
self.api_key = api_key
self.headers = {
"x-goog-api-key": api_key,
"Content-Type": "application/json"
}
self.headers = {"x-goog-api-key": api_key, "Content-Type": "application/json"}
def generate_video(self, prompt: str) -> Optional[str]:
"""
Initiate video generation with Veo.
Args:
prompt: Text description of the video to generate
Returns:
Operation name if successful, None otherwise
"""
print(f"🎬 Generating video with prompt: '{prompt}'")
url = f"{self.base_url}/models/veo-3.0-generate-preview:predictLongRunning"
payload = {
"instances": [{
"prompt": prompt
}]
}
payload = {"instances": [{"prompt": prompt}]}
try:
response = requests.post(url, headers=self.headers, json=payload)
response.raise_for_status()
data = response.json()
operation_name = data.get("name")
if operation_name:
print(f"✅ Video generation started: {operation_name}")
return operation_name
@ -71,58 +67,64 @@ class VeoVideoGenerator:
print("❌ No operation name returned")
print(f"Response: {json.dumps(data, indent=2)}")
return None
except requests.RequestException as e:
print(f"❌ Failed to start video generation: {e}")
if hasattr(e, 'response') and e.response is not None:
if hasattr(e, "response") and e.response is not None:
try:
error_data = e.response.json()
print(f"Error details: {json.dumps(error_data, indent=2)}")
except:
print(f"Error response: {e.response.text}")
return None
def wait_for_completion(self, operation_name: str, max_wait_time: int = 600) -> Optional[str]:
def wait_for_completion(
self, operation_name: str, max_wait_time: int = 600
) -> Optional[str]:
"""
Poll operation status until video generation is complete.
Args:
operation_name: Name of the operation to monitor
max_wait_time: Maximum time to wait in seconds (default: 10 minutes)
Returns:
Video URI if successful, None otherwise
"""
print("⏳ Waiting for video generation to complete...")
operation_url = f"{self.base_url}/{operation_name}"
start_time = time.time()
poll_interval = 10 # Start with 10 seconds
while time.time() - start_time < max_wait_time:
try:
print(f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)")
print(
f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)"
)
response = requests.get(operation_url, headers=self.headers)
response.raise_for_status()
data = response.json()
# Check for errors
if "error" in data:
print("❌ Error in video generation:")
print(json.dumps(data["error"], indent=2))
return None
# Check if operation is complete
is_done = data.get("done", False)
if is_done:
print("🎉 Video generation complete!")
try:
# Extract video URI from nested response
video_uri = data["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"]
video_uri = data["response"]["generateVideoResponse"][
"generatedSamples"
][0]["video"]["uri"]
print(f"📹 Video URI: {video_uri}")
return video_uri
except KeyError as e:
@ -130,64 +132,68 @@ class VeoVideoGenerator:
print("Full response:")
print(json.dumps(data, indent=2))
return None
# Wait before next poll, with exponential backoff
time.sleep(poll_interval)
poll_interval = min(poll_interval * 1.2, 30) # Cap at 30 seconds
except requests.RequestException as e:
print(f"❌ Error polling operation status: {e}")
time.sleep(poll_interval)
print(f"⏰ Timeout after {max_wait_time} seconds")
return None
def download_video(self, video_uri: str, output_filename: str = "generated_video.mp4") -> bool:
def download_video(
self, video_uri: str, output_filename: str = "generated_video.mp4"
) -> bool:
"""
Download the generated video file.
Args:
video_uri: URI of the video to download (from Google's response)
output_filename: Local filename to save the video
Returns:
True if download successful, False otherwise
"""
print(f"⬇️ Downloading video...")
print(f"Original URI: {video_uri}")
# Convert Google URI to LiteLLM proxy URI
# Example: files/abc123 -> /gemini/v1beta/files/abc123:download?alt=media
if video_uri.startswith("files/"):
download_path = f"{video_uri}:download?alt=media"
else:
download_path = video_uri
litellm_download_url = f"{self.base_url}/{download_path}"
print(f"Download URL: {litellm_download_url}")
try:
# Download with streaming and redirect handling
response = requests.get(
litellm_download_url,
headers=self.headers,
litellm_download_url,
headers=self.headers,
stream=True,
allow_redirects=True # Handle redirects automatically
allow_redirects=True, # Handle redirects automatically
)
response.raise_for_status()
# Save video file
with open(output_filename, 'wb') as f:
with open(output_filename, "wb") as f:
downloaded_size = 0
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
downloaded_size += len(chunk)
# Progress indicator for large files
if downloaded_size % (1024 * 1024) == 0: # Every MB
print(f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB...")
print(
f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB..."
)
# Verify file was created and has content
if os.path.exists(output_filename):
file_size = os.path.getsize(output_filename)
@ -203,48 +209,52 @@ class VeoVideoGenerator:
else:
print("❌ File was not created")
return False
except requests.RequestException as e:
print(f"❌ Download failed: {e}")
if hasattr(e, 'response') and e.response is not None:
if hasattr(e, "response") and e.response is not None:
print(f"Status code: {e.response.status_code}")
print(f"Response headers: {dict(e.response.headers)}")
return False
def generate_and_download(self, prompt: str, output_filename: str = None) -> bool:
"""
Complete workflow: generate video and download it.
Args:
prompt: Text description for video generation
output_filename: Output filename (auto-generated if None)
Returns:
True if successful, False otherwise
"""
# Auto-generate filename if not provided
if output_filename is None:
timestamp = int(time.time())
safe_prompt = "".join(c for c in prompt[:30] if c.isalnum() or c in (' ', '-', '_')).rstrip()
output_filename = f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
safe_prompt = "".join(
c for c in prompt[:30] if c.isalnum() or c in (" ", "-", "_")
).rstrip()
output_filename = (
f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
)
print("=" * 60)
print("🎬 VEO VIDEO GENERATION WORKFLOW")
print("=" * 60)
# Step 1: Generate video
operation_name = self.generate_video(prompt)
if not operation_name:
return False
# Step 2: Wait for completion
video_uri = self.wait_for_completion(operation_name)
if not video_uri:
return False
# Step 3: Download video
success = self.download_video(video_uri, output_filename)
if success:
print("=" * 60)
print("🎉 SUCCESS! Video generation complete!")
@ -254,51 +264,51 @@ class VeoVideoGenerator:
print("=" * 60)
print("❌ FAILED! Video generation or download failed")
print("=" * 60)
return success
def main():
"""
Example usage of the VeoVideoGenerator.
Configure these environment variables:
- LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta)
- LITELLM_API_KEY: Your LiteLLM API key (default: sk-1234)
"""
# Configuration from environment or defaults
base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta")
api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
print("🚀 Starting Veo Video Generation Example")
print(f"📡 Using LiteLLM proxy at: {base_url}")
# Initialize generator
generator = VeoVideoGenerator(base_url=base_url, api_key=api_key)
# Example prompts - try different ones!
example_prompts = [
"A cat playing with a ball of yarn in a sunny garden",
"Ocean waves crashing against rocky cliffs at sunset",
"A bustling city street with people walking and cars passing by",
"A peaceful forest with sunlight filtering through the trees"
"A peaceful forest with sunlight filtering through the trees",
]
# Use first example or get from user
prompt = example_prompts[0]
print(f"🎬 Using prompt: '{prompt}'")
# Generate and download video
success = generator.generate_and_download(prompt)
if success:
print("\n✅ Example completed successfully!")
print("💡 Try modifying the prompt in the script for different videos!")
else:
print("\n❌ Example failed!")
print("🔧 Check your LiteLLM proxy configuration and Google AI Studio API key")
# Troubleshooting tips
print("\n🔍 Troubleshooting:")
print("1. Ensure LiteLLM proxy is running with Google AI Studio pass-through")

4
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -47,7 +47,7 @@ spec:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
{{- with .Values.extraInitContainers }}
initContainers:
{{- toYaml . | nindent 8 }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
containers:
- name: {{ include "litellm.name" . }}
@ -212,7 +212,7 @@ spec:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.extraContainers }}
{{- toYaml . | nindent 8 }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
volumes:
{{ if .Values.securityContext.readOnlyRootFilesystem }}

4
deploy/charts/litellm-helm/templates/migrations-job.yaml
View file

@ -37,7 +37,7 @@ spec:
serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
{{- with .Values.migrationJob.extraInitContainers }}
initContainers:
{{- toYaml . | nindent 8 }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
containers:
- name: prisma-migrations
@ -96,7 +96,7 @@ spec:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.migrationJob.extraContainers }}
{{- toYaml . | nindent 8 }}
{{- tpl (toYaml .) $ | nindent 8 }}
{{- end }}
{{- with .Values.volumes }}
volumes:

58
deploy/charts/litellm-helm/tests/deployment_tests.yaml
View file

@ -319,3 +319,61 @@ tests:
asserts:
- notExists:
path: spec.minReadySeconds
- it: should work with extraInitContainers
template: deployment.yaml
set:
extraInitContainers:
- name: init-test
image: busybox:latest
command: ["echo", "hello"]
asserts:
- contains:
path: spec.template.spec.initContainers
content:
name: init-test
image: busybox:latest
command: ["echo", "hello"]
- it: should support tpl in extraInitContainers
template: deployment.yaml
set:
image:
repository: ghcr.io/berriai/litellm-database
tag: test
extraInitContainers:
- name: init-tpl
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
command: ["echo", "hello"]
asserts:
- contains:
path: spec.template.spec.initContainers
content:
name: init-tpl
image: "ghcr.io/berriai/litellm-database:test"
command: ["echo", "hello"]
- it: should work with extraContainers
template: deployment.yaml
set:
extraContainers:
- name: sidecar
image: busybox:latest
asserts:
- contains:
path: spec.template.spec.containers
content:
name: sidecar
image: busybox:latest
- it: should support tpl in extraContainers
template: deployment.yaml
set:
image:
repository: ghcr.io/berriai/litellm-database
tag: test
extraContainers:
- name: sidecar-tpl
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
asserts:
- contains:
path: spec.template.spec.containers
content:
name: sidecar-tpl
image: "ghcr.io/berriai/litellm-database:test"

66
deploy/charts/litellm-helm/tests/migrations-job_tests.yaml
View file

@ -188,3 +188,69 @@ tests:
- equal:
path: spec.template.spec.serviceAccountName
value: pre-existing-sa
- it: should work with extraInitContainers
template: migrations-job.yaml
set:
migrationJob:
enabled: true
extraInitContainers:
- name: init-test
image: busybox:latest
command: ["echo", "hello"]
asserts:
- contains:
path: spec.template.spec.initContainers
content:
name: init-test
image: busybox:latest
command: ["echo", "hello"]
- it: should support tpl in extraInitContainers
template: migrations-job.yaml
set:
image:
repository: ghcr.io/berriai/litellm-database
tag: test
migrationJob:
enabled: true
extraInitContainers:
- name: init-tpl
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
command: ["echo", "hello"]
asserts:
- contains:
path: spec.template.spec.initContainers
content:
name: init-tpl
image: "ghcr.io/berriai/litellm-database:test"
command: ["echo", "hello"]
- it: should work with extraContainers
template: migrations-job.yaml
set:
migrationJob:
enabled: true
extraContainers:
- name: sidecar
image: busybox:latest
asserts:
- contains:
path: spec.template.spec.containers
content:
name: sidecar
image: busybox:latest
- it: should support tpl in extraContainers
template: migrations-job.yaml
set:
image:
repository: ghcr.io/berriai/litellm-database
tag: test
migrationJob:
enabled: true
extraContainers:
- name: sidecar-tpl
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
asserts:
- contains:
path: spec.template.spec.containers
content:
name: sidecar-tpl
image: "ghcr.io/berriai/litellm-database:test"

117
docker/Dockerfile.non_root
View file

@ -15,29 +15,21 @@ COPY --from=uvbin /uv /usr/local/bin/uv
COPY --from=uvbin /uvx /usr/local/bin/uvx
RUN for i in 1 2 3; do \
apk add --no-cache \
python3 \
python3-dev \
clang \
llvm \
lld \
gcc \
linux-headers \
build-base \
bash \
coreutils \
curl \
openssl \
openssl-dev \
nodejs \
npm \
libsndfile && break || sleep 5; \
apk add --no-cache \
python3 \
python3-dev \
gcc \
bash \
coreutils \
curl \
openssl \
libsndfile \
nodejs && break || sleep 5; \
done
ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
UV_LINK_MODE=copy \
NVM_DIR=/root/.nvm \
PATH="/root/.nvm/versions/node/v20.20.2/bin:/app/.venv/bin:${PATH}" \
PATH="/app/.venv/bin:${PATH}" \
LITELLM_NON_ROOT=true \
PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
@ -49,7 +41,8 @@ COPY enterprise/pyproject.toml enterprise/
COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
--extra extra_proxy \
@ -62,38 +55,12 @@ COPY . .
# Set non-root flag for build time consistency
ENV LITELLM_NON_ROOT=true
# Build Admin UI once and stage the static output for the runtime image.
# NOTE: .npmrc files (which may set ignore-scripts=true and min-release-age=3d)
# are temporarily renamed during npm install/ci so they don't block lifecycle
# scripts needed by the build. This is safe because npm ci installs from
# package-lock.json with pinned versions + integrity hashes.
# Stage the pre-built Admin UI from the checked-in Next.js static export.
# _experimental/out/ is regenerated as part of the release runbook.
# Restructure extensionless routes (foo.html -> foo/index.html) to match the layout
# proxy_server.py expects, and drop a readiness marker.
RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
([ -f /app/.npmrc ] && mv /app/.npmrc /app/.npmrc.bak || true) && \
NVM_VERSION="v0.40.4" && \
NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" && \
NODE_VERSION="v20.20.2" && \
NVM_SCRIPT="/tmp/install-nvm.sh" && \
curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT" && \
echo "${NVM_CHECKSUM} ${NVM_SCRIPT}" | sha256sum -c - && \
bash "$NVM_SCRIPT" && \
export NVM_DIR="$HOME/.nvm" && \
. "$NVM_DIR/nvm.sh" && \
nvm install "${NODE_VERSION}" && \
nvm use "${NODE_VERSION}" && \
npm install -g npm@11.12.1 && \
npm install -g node-gyp@12.2.0 && \
ln -sf "$(npm root -g)/node-gyp" "$(npm root -g)/npm/node_modules/node-gyp" && \
npm cache clean --force && \
cd /app/ui/litellm-dashboard && \
if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
fi && \
([ -f .npmrc ] && mv .npmrc .npmrc.bak || true) && \
npm ci --no-audit --no-fund && \
([ -f .npmrc.bak ] && mv .npmrc.bak .npmrc || true) && \
([ -f /app/.npmrc.bak ] && mv /app/.npmrc.bak /app/.npmrc || true) && \
npm run build && \
cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \
cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
( cd /var/lib/litellm/ui && \
for html_file in *.html; do \
@ -103,10 +70,10 @@ RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
mv "$html_file" "$folder_name/index.html"; \
fi; \
done && \
touch .litellm_ui_ready ) && \
cd /app/ui/litellm-dashboard && rm -rf ./out
touch .litellm_ui_ready )
RUN if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
uv sync --frozen --no-default-groups --no-editable \
--extra proxy \
--extra proxy-runtime \
@ -123,10 +90,7 @@ RUN if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
--python python3; \
fi
RUN mkdir -p /app/.cache/npm && \
prisma generate --schema=./schema.prisma && \
prisma --version && \
prisma migrate diff --from-empty --to-schema-datamodel ./schema.prisma --script > /dev/null 2>&1 || true
RUN prisma generate --schema=./schema.prisma
RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@ -137,33 +101,11 @@ WORKDIR /app
USER root
RUN for i in 1 2 3; do \
apk upgrade --no-cache && break || sleep 5; \
apk upgrade --no-cache && break || sleep 5; \
done && \
for i in 1 2 3; do \
apk add --no-cache python3 bash openssl tzdata nodejs npm supervisor libsndfile && break || sleep 5; \
done && \
apk upgrade --no-cache nodejs && \
npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
GLOBAL="$(npm root -g)" && \
find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
done && \
find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
done && \
find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
npm cache clean --force && \
{ apk del --no-cache npm 2>/dev/null || true; }
apk add --no-cache python3 bash openssl tzdata supervisor libsndfile nodejs && break || sleep 5; \
done
COPY --from=builder /app /app
COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
@ -179,15 +121,10 @@ ENV PATH="/app/.venv/bin:${PATH}" \
PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
PRISMA_HIDE_UPDATE_MESSAGE=1 \
PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
NPM_CONFIG_CACHE=/app/.cache/npm \
NPM_CONFIG_PREFER_OFFLINE=true \
PRISMA_OFFLINE_MODE=true
RUN sed -i 's/\r$//' docker/entrypoint.sh && \
sed -i 's/\r$//' docker/prod_entrypoint.sh && \
chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui /tmp/.npm && \
chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm /tmp/.npm && \
RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \
chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \
PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
chown -R nobody:nogroup "$PRISMA_PATH" && \
LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \

1
docs/my-website/docs/completion/prompt_caching.md
View file

@ -10,6 +10,7 @@ Supported Providers:
- Vertex AI (`vertex_ai/`, `vertex_ai_beta/`)
- Bedrock (`bedrock/`, `bedrock/invoke/`, `bedrock/converse`) ([All models bedrock supports prompt caching on](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html))
- Deepseek API (`deepseek/`)
- xAI (`xai/`)
For the supported providers, LiteLLM follows the OpenAI prompt caching usage object format:

25
docs/my-website/docs/completion/prompt_compression.md
View file

@ -8,6 +8,7 @@ The function keeps high-relevance and recent context, replaces low-relevance con
```python
import litellm
from litellm.types.utils import CallTypes
messages = [
{"role": "system", "content": "You are a coding assistant."},
@ -19,6 +20,7 @@ messages = [
compressed = litellm.compress(
messages=messages,
model="gpt-4o",
call_type=CallTypes.completion,
compression_trigger=1000,
compression_target=500,
)
@ -45,6 +47,7 @@ response = litellm.completion(
- `messages` (`List[dict]`, required): input conversation messages
- `model` (`str`, required): model name used for token counting
- `call_type` (`CallTypes`, default `CallTypes.completion`): the LiteLLM call type whose message schema these messages follow. Supported values: `CallTypes.completion` / `CallTypes.acompletion` (OpenAI chat-completions shape) and `CallTypes.anthropic_messages` (Anthropic Messages shape)
- `compression_trigger` (`int`, default `200000`): compress only if input token count exceeds this
- `compression_target` (`Optional[int]`, default `70% of compression_trigger`): desired post-compression token budget
- `embedding_model` (`Optional[str]`): if set, combines BM25 + embedding relevance scoring
@ -70,6 +73,28 @@ args = json.loads(tool_call.function.arguments)
full_content = compressed["cache"][args["key"]]
```
## Server-side Callback Loop (`/v1/messages`)
You can enable callback-based compression interception to make retrieval loops
transparent for Anthropic Messages calls:
```yaml
litellm_settings:
callbacks: ["compression_interception"]
compression_interception_params:
enabled: true
compression_trigger: 10000
compression_target: 7000
```
With this enabled, LiteLLM runs the following server-side flow:
1. Compresses inbound messages before the first provider call.
2. Injects the `litellm_content_retrieve` tool.
3. Detects retrieval `tool_use` blocks in the model response.
4. Resolves retrieval keys from the compression cache.
5. Reruns the model via agentic loop and returns the final answer.
## Performance
Benchmarked on [SWE-bench Lite](https://huggingface.co/datasets/princeton-nlp/SWE-bench_Lite_bm25_27K) (real GitHub issues with ~27k tokens of BM25-retrieved repo context per problem).

41
docs/my-website/docs/providers/scaleway.md
View file

@ -60,3 +60,44 @@ curl http://localhost:4000/chat/completions \
## Supported features
Scaleway provider supports all features in [Generative APIs reference documentation ↗](https://www.scaleway.com/en/developers/api/generative-apis/), such as streaming, structured outputs and tool calling.
## Audio transcription
Scaleway's `/audio/transcriptions` endpoint is OpenAI-compatible and works with Whisper models.
### Python SDK
```python
import os
from litellm import transcription
os.environ["SCW_SECRET_KEY"] = "your-scaleway-secret-key"
with open("speech.mp3", "rb") as audio_file:
response = transcription(
model="scaleway/whisper-large-v3",
file=audio_file,
)
print(response.text)
```
### Proxy config
```yaml
model_list:
- model_name: scaleway-whisper
litellm_params:
model: scaleway/whisper-large-v3
api_key: "os.environ/SCW_SECRET_KEY"
```
### Proxy request
```bash
curl http://localhost:4000/v1/audio/transcriptions \
-H "Authorization: Bearer YOUR_LITELLM_MASTER_KEY" \
-F model="scaleway-whisper" \
-F file="@speech.mp3"
```
Supported optional params: `language`, `prompt`, `response_format`, `temperature`, `timestamp_granularities`.

95
docs/my-website/docs/proxy/agentic_loop_hook.md Normal file
View file

@ -0,0 +1,95 @@
# Agentic Loop Hook
Build a `CustomLogger` callback that intercepts a model response, fulfills tool calls server-side, and reruns the model — transparently to the caller.
:::info Supported call types
- `async` only (sync calls do not trigger the hook)
- Non-streaming only (streaming responses cannot be inspected for tool calls)
- Works on both `/v1/messages` and `/v1/chat/completions`
:::
## Implement the callback
Override two methods on `CustomLogger`:
```python
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch
MY_TOOL = "my_tool"
class MyToolCallback(CustomLogger):
async def async_should_run_agentic_loop(
self, response, model, messages, tools, stream, custom_llm_provider, kwargs
):
# Return (True, context_dict) if there are tool calls to handle
content = getattr(response, "content", None) or []
calls = [b for b in content if isinstance(b, dict)
and b.get("type") == "tool_use" and b.get("name") == MY_TOOL]
if not calls:
return False, {}
return True, {"tool_calls": calls}
async def async_build_agentic_loop_plan(
self, tools, model, messages, response,
anthropic_messages_provider_config,
anthropic_messages_optional_request_params,
logging_obj, stream, kwargs,
):
calls = tools["tool_calls"]
results = [f"result for {c['input']}" for c in calls] # your logic here
follow_up = messages + [
{"role": "assistant", "content": [
{"type": "tool_use", "id": c["id"], "name": c["name"], "input": c["input"]}
for c in calls
]},
{"role": "user", "content": [
{"type": "tool_result", "tool_use_id": c["id"], "content": results[i]}
for i, c in enumerate(calls)
]},
]
return AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(messages=follow_up),
)
```
For `/v1/chat/completions`, override `async_build_chat_completion_agentic_loop_plan` instead — same idea, `optional_params` replaces `anthropic_messages_optional_request_params`.
## Register it
```python
import litellm
litellm.callbacks = [MyToolCallback()]
```
Or in `config.yaml`:
```yaml
litellm_settings:
callbacks: ["my_module.MyToolCallback"]
```
## `AgenticLoopPlan` fields
| Field | Effect |
|---|---|
| `run_agentic_loop=True` + `request_patch` | Reruns the model with the patched request |
| `response_override` | Returns this value directly to the caller (no rerun) |
| `terminate=True` | Stops the loop, returns the current response |
| `run_agentic_loop=False` (default) | Skips; next callback is checked |
`AgenticLoopRequestPatch` accepts: `model`, `messages`, `tools`, `max_tokens`, `optional_params`, `kwargs`.
## Loop safety
- Default max reruns: `3` — override per-request with `kwargs["max_agentic_loops"]`
- Identical tool-call fingerprints abort the loop automatically
- Current depth is in `kwargs["_agentic_loop_depth"]`
## Examples in this repo
- `litellm/integrations/compression_interception/handler.py`
- `litellm/integrations/websearch_interception/handler.py`

3
docs/my-website/docs/proxy/config_settings.md
View file

@ -487,7 +487,8 @@ router_settings:
| AZURE_STORAGE_CLIENT_ID | The Application Client ID to use for Authentication to Azure Blob Storage logging
| AZURE_STORAGE_CLIENT_SECRET | The Application Client Secret to use for Authentication to Azure Blob Storage logging
| AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY | Cost per GB per day for Azure Vector Store service
| BACKGROUND_HEALTH_CHECK_MAX_TOKENS | Optional global default for `max_tokens` on proxy background health checks when a model has no `health_check_max_tokens`. If unset, non-wildcard models default to 1. Applies to wildcard routes when set. Default is unset
| BACKGROUND_HEALTH_CHECK_MAX_TOKENS | Optional global default for `max_tokens` on proxy background health checks when a model has no `health_check_max_tokens`. If unset, non-wildcard models default to 5. Applies to wildcard routes when set. Default is unset
| BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING | For **non-wildcard** reasoning models (`supports_reasoning(model)=true`), this takes precedence over `BACKGROUND_HEALTH_CHECK_MAX_TOKENS` when set. If unset, reasoning models fall back to `BACKGROUND_HEALTH_CHECK_MAX_TOKENS` (if set) or default behavior. Wildcard routes ignore this. Default is unset
| BATCH_STATUS_POLL_INTERVAL_SECONDS | Interval in seconds for polling batch status. Default is 3600 (1 hour)
| BATCH_STATUS_POLL_MAX_ATTEMPTS | Maximum number of attempts for polling batch status. Default is 24 (for 24 hours)
| BEDROCK_MAX_POLICY_SIZE | Maximum size for Bedrock policy. Default is 75

26
docs/my-website/docs/proxy/health.md
View file

@ -338,7 +338,7 @@ model_list:
## Health Check Max Tokens
By default, health checks use `max_tokens=1` to minimize cost and latency. For wildcard models, the default is `max_tokens=10`.
By default, health checks use `max_tokens=5` to balance reliability with low cost and latency. For wildcard models, the default is `max_tokens=10`.
You can override this per-model by setting `health_check_max_tokens` in the `model_info` section of your config.yaml.
@ -352,6 +352,30 @@ model_list:
health_check_max_tokens: 5 # 👈 OVERRIDE HEALTH CHECK MAX TOKENS
```
### Reasoning vs non-reasoning defaults
Reasoning models (per `supports_reasoning` in the model map) often need a higher health-check `max_tokens` because providers count reasoning tokens toward the completion budget. You can set **separate** limits without listing every model:
**Per deployment (`model_info`)** — used when `health_check_max_tokens` is not set. Ignored for wildcard routes (`*` in `litellm_params.model`, i.e. the deployment model string; not `health_check_model`).
```yaml
model_list:
- model_name: openai-stack
litellm_params:
model: openai/gpt-5-nano
api_key: os.environ/OPENAI_API_KEY
model_info:
health_check_max_tokens_reasoning: 128
health_check_max_tokens_non_reasoning: 1
```
**Global (environment)**:
- `BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING` — for non-wildcard reasoning models, this value takes precedence when set
- `BACKGROUND_HEALTH_CHECK_MAX_TOKENS` — global fallback for all models (including wildcard routes)
If neither is set, non-wildcard models default to `5` and wildcard routes omit `max_tokens`.
## `/health/readiness`
Unprotected endpoint for checking if proxy is ready to accept requests

61
docs/my-website/docs/proxy/users.md
View file

@ -333,6 +333,67 @@ curl 'http://0.0.0.0:4000/key/generate' \
}'
```
#### **Set multiple budget windows on a key**
Apply multiple concurrent budget limits at different time scales on the same key — for example, cap a key at **$10/day** AND **$100/month**.
**When is this useful?**
A single `budget_duration` window can't prevent a bad day from burning your entire month. Multiple budget windows let you:
- Block a runaway usage spike within the day while still allowing normal monthly spend.
- Give Claude Code rollouts a daily guardrail (`24h`) and a monthly ceiling (`30d`) so a single heavy session doesn't exhaust the whole month.
- Layer fine-grained hourly limits for bursty workloads on top of a weekly cap.
:::info
See [User Budget docs](https://docs.litellm.ai/docs/proxy/users) for more on how budgets work across keys, teams, and users.
:::
**Via API**
Pass `budget_limits` as a list of `{budget_duration, max_budget}` objects:
```bash
curl 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer <your-master-key>' \
--header 'Content-Type: application/json' \
--data-raw '{
"budget_limits": [
{"budget_duration": "24h", "max_budget": 10},
{"budget_duration": "30d", "max_budget": 100}
]
}'
```
Each window is tracked independently and resets on its own schedule:
| `budget_duration` | Resets |
|---|---|
| `1h` | Every hour |
| `24h` | Daily at midnight UTC |
| `7d` | Every Sunday at midnight UTC |
| `30d` | 1st of every month at midnight UTC |
**Via Dashboard**
Open **Virtual Keys → Create Key → Optional Settings → Budget Windows**.
![Step 1 - open key settings](https://colony-recorder.s3.amazonaws.com/files/2026-04-01/18930ba5-67c0-4031-afc0-57f37b4e59e4/ascreenshot_ef79d8a000bb41cdacf1bd9827732ee8_text_export.jpeg)
Click **+ Add Budget Window** to add a row, choose the period from the dropdown, and enter the spend cap.
![Step 2 - add a window](https://colony-recorder.s3.amazonaws.com/files/2026-04-01/5ae8c0b3-2d03-41ad-a63c-47b20c350dfe/ascreenshot_1a7dc6c7d65544f38fd8a65604674f22_text_export.jpeg)
Add a second row for a different time period (e.g. monthly $100 on top of a daily $10).
![Step 3 - add second window](https://colony-recorder.s3.amazonaws.com/files/2026-04-01/cbded3a7-1086-4e20-8f0f-de154b76146c/ascreenshot_c51c18752c3b4f8b976d28799b2638b6_text_export.jpeg)
Each window shows the reset schedule below the input so it's always clear when spend resets.
![Step 4 - reset hints](https://colony-recorder.s3.amazonaws.com/files/2026-04-01/8754f121-1640-4892-9dd0-fd4a870418bf/ascreenshot_8079eb0df2194e8f99e5258ba4b3c082_text_export.jpeg)
### ✨ Virtual Key (Model Specific)

111
docs/my-website/docs/skills_gateway.md Normal file
View file

@ -0,0 +1,111 @@
# Skills Gateway
<iframe width="840" height="500" src="https://www.loom.com/embed/cb74eb79df3e4c2b83a6efae54a589f9" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
LiteLLM acts as a **Skills Registry** — a central place to register, manage, and discover Claude Code skills across your organization. Teams can publish skills once and have agents and developers find them through a single hub.
## How it works
```mermaid
graph TD
Dev["👨‍💻 Developer<br/>registers a skill<br/>(GitHub URL or subdir)"] -->|POST /claude-code/plugins| Proxy["LiteLLM Proxy<br/>(Skills Registry)"]
Admin["🔑 Admin<br/>publishes skill<br/>(marks as public)"] -->|enable via UI or API| Proxy
Proxy -->|GET /public/skill_hub| SkillHub["🗂️ Skill Hub<br/>(AI Hub → Skill Hub tab)"]
Proxy -->|GET /claude-code/marketplace.json| Marketplace["📦 Claude Code<br/>Marketplace endpoint"]
SkillHub --> Human["🧑 Human<br/>browses & discovers skills<br/>in AI Hub UI"]
Marketplace --> Agent["🤖 Agent / Claude Code<br/>installs skill with<br/>/plugin marketplace add &lt;name&gt;"]
style Proxy fill:#1a73e8,color:#fff
style SkillHub fill:#e8f0fe,color:#1a73e8
style Marketplace fill:#e8f0fe,color:#1a73e8
```
## Quick start
### 1. Register a skill
Paste any GitHub URL into the Skills UI — LiteLLM auto-detects the source type and skill name.
```bash
curl -X POST https://your-proxy/claude-code/plugins \
-H "Authorization: Bearer $LITELLM_KEY" \
-H "Content-Type: application/json" \
-d '{
"name": "grill-me",
"source": {
"source": "git-subdir",
"url": "https://github.com/mattpocock/skills",
"path": "grill-me"
},
"description": "Interview skill for relentless questioning",
"domain": "Productivity",
"namespace": "interviews"
}'
```
Skills nested in subdirectories (e.g. `github.com/org/repo/tree/main/skill-name`) are supported — LiteLLM parses the URL automatically in the UI.
### 2. Publish to hub
In the Admin UI: **AI Hub → Skill Hub → Select Skills to Make Public**.
Or via API:
```bash
curl -X POST https://your-proxy/claude-code/plugins/grill-me/enable \
-H "Authorization: Bearer $LITELLM_KEY"
```
### 3. Browse the hub
Public skills appear at:
- **Admin UI**: AI Hub → Skill Hub tab
- **Public page**: `/ui/model_hub` → Skill Hub tab (no login required)
- **API**: `GET /public/skill_hub`
### 4. Install in Claude Code
Point Claude Code at your proxy marketplace once:
```json title="~/.claude/settings.json"
{
"extraKnownMarketplaces": {
"my-org": {
"source": "url",
"url": "https://your-proxy/claude-code/marketplace.json"
}
}
}
```
Then install any skill:
```
/plugin marketplace add grill-me
```
## Skill fields
| Field | Description |
|-------|-------------|
| `name` | Unique skill identifier (used in `/plugin marketplace add`) |
| `source` | Git source — `github`, `url`, or `git-subdir` |
| `description` | Short description shown in the hub |
| `domain` | Category for grouping (e.g. `Engineering`, `Productivity`) |
| `namespace` | Subcategory within a domain (e.g. `quality`, `meetings`) |
| `keywords` | Tags for search and filtering |
| `version` | Semver string |
## API reference
| Endpoint | Auth | Description |
|----------|------|-------------|
| `POST /claude-code/plugins` | Required | Register a skill |
| `GET /claude-code/plugins` | Required | List all skills (admin) |
| `POST /claude-code/plugins/{name}/enable` | Required | Publish a skill |
| `POST /claude-code/plugins/{name}/disable` | Required | Unpublish a skill |
| `GET /public/skill_hub` | None | List public skills |
| `GET /claude-code/marketplace.json` | None | Claude Code marketplace manifest |

11
docs/my-website/docs/tutorials/claude_code_byok.md
View file

@ -35,6 +35,17 @@ By default, LiteLLM strips `x-api-key` from client requests for security. Settin
:::
:::tip Configure via UI instead of config.yaml
You can also complete this setup from the LiteLLM admin UI:
- Add the model via **Models → Add Model**, leaving the **API Key** field blank.
- Enable the toggle at **Settings → UI Settings → "Forward LLM provider auth headers"**.
Both UI actions write to the database and override `config.yaml` at runtime.
:::
## Step 2: Create a LiteLLM Virtual Key
Create a virtual key in the LiteLLM UI or via API.

16
docs/my-website/docs/tutorials/prompt_caching.md
View file

@ -8,6 +8,22 @@ Reduce costs by up to 90% by using LiteLLM to auto-inject prompt caching checkpo
<Image img={require('../../img/auto_prompt_caching.png')} style={{ width: '800px', height: 'auto' }} />
Supported Providers (`cache_control` marker):
- Anthropic API (`anthropic/`)
- AWS Bedrock - Claude (`bedrock/`)
- Vertex AI - Claude and Gemini (`vertex_ai/`)
- Google AI Studio - Gemini (`gemini/`)
- Azure AI - Claude (`azure_ai/`)
- OpenRouter - Claude, Gemini, MiniMax, GLM, z-ai routes (`openrouter/`)
- Databricks - Claude (`databricks/`)
- DashScope / Qwen (`dashscope/`)
- MiniMax (`minimax/`)
- Z.ai / GLM (`zai/`)
Provider Managed (automatic, no marker needed):
- OpenAI (`openai/`)
- DeepSeek (`deepseek/`)
- xAI (`xai/`)
## How it works

28
docs/my-website/docusaurus.config.js
View file

@ -187,6 +187,32 @@ const config = {
},
],
[
'@signalwire/docusaurus-plugin-llms-txt',
{
markdown: {
enableFiles: true,
includeDocs: true,
},
llmsTxt: {
enableLlmsFullTxt: true,
includeDocs: true,
},
ui: {
copyPageContent: {
buttonLabel: 'Copy Page',
actions: {
viewMarkdown: true,
ai: {
chatGPT: true,
claude: true,
},
},
},
},
},
],
() => ({
name: 'cripchat',
injectHtmlTags() {
@ -239,7 +265,7 @@ const config = {
],
],
themes: ['@docusaurus/theme-mermaid'],
themes: ['@docusaurus/theme-mermaid', '@signalwire/docusaurus-theme-llms-txt'],
markdown: {
mermaid: true,
},

396
docs/my-website/package-lock.json generated
View file

@ -15,6 +15,8 @@
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "0.5.107",
"@mdx-js/react": "3.1.1",
"@signalwire/docusaurus-plugin-llms-txt": "2.0.0-alpha.7",
"@signalwire/docusaurus-theme-llms-txt": "1.0.0-alpha.9",
"clsx": "1.2.1",
"prism-react-renderer": "1.3.5",
"react": "18.3.1",
@ -7140,6 +7142,72 @@
"integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==",
"license": "BSD-3-Clause"
},
"node_modules/@signalwire/docusaurus-plugin-llms-txt": {
"version": "2.0.0-alpha.7",
"resolved": "https://registry.npmjs.org/@signalwire/docusaurus-plugin-llms-txt/-/docusaurus-plugin-llms-txt-2.0.0-alpha.7.tgz",
"integrity": "sha512-v9EcYXVNvMydIWVIzI1H2iC4/BNdystE0jJAQIFu68SHy1a13dESz9hn5YJE9Izx18QPny1jhXym/3wEP9+8LA==",
"license": "MIT",
"dependencies": {
"fs-extra": "^11.0.0",
"hast-util-select": "^6.0.4",
"hast-util-to-html": "^9.0.5",
"hast-util-to-string": "^3.0.1",
"p-map": "^7.0.2",
"rehype-parse": "^9",
"rehype-remark": "^10",
"remark-gfm": "^4",
"remark-stringify": "^11",
"string-width": "^5.0.0",
"unified": "^11",
"unist-util-visit": "^5"
},
"engines": {
"node": ">=18.0.0"
},
"peerDependencies": {
"@docusaurus/core": "^3.0.0"
}
},
"node_modules/@signalwire/docusaurus-plugin-llms-txt/node_modules/p-map": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.4.tgz",
"integrity": "sha512-tkAQEw8ysMzmkhgw8k+1U/iPhWNhykKnSk4Rd5zLoPJCuJaGRPo6YposrZgaxHKzDHdDWWZvE/Sk7hsL2X/CpQ==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/@signalwire/docusaurus-theme-llms-txt": {
"version": "1.0.0-alpha.9",
"resolved": "https://registry.npmjs.org/@signalwire/docusaurus-theme-llms-txt/-/docusaurus-theme-llms-txt-1.0.0-alpha.9.tgz",
"integrity": "sha512-ULCKEKkAUZVnLr8+ocR4tl7ogiiW13Hqtoo8SfNbgOyX1l4LN3a6j3/vxgc6qRYbgWlnqY3EPC7S3tenRsDjgQ==",
"license": "MIT",
"dependencies": {
"@docusaurus/core": "^3.0.0",
"@docusaurus/theme-common": "^3.0.0",
"clsx": "^2.0.0",
"react-icons": "^5.5.0"
},
"engines": {
"node": ">=18.0.0"
},
"peerDependencies": {
"react": "^18.0.0",
"react-dom": "^18.0.0"
}
},
"node_modules/@signalwire/docusaurus-theme-llms-txt/node_modules/clsx": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
"integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/@sinclair/typebox": {
"version": "0.27.10",
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.10.tgz",
@ -8972,6 +9040,16 @@
"integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==",
"license": "MIT"
},
"node_modules/bcp-47-match": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/bcp-47-match/-/bcp-47-match-2.0.3.tgz",
"integrity": "sha512-JtTezzbAibu8G0R9op9zb3vcWZd9JF6M0xOYGPn0fNCd7wOpRB1mU2mH9T8gaBGbAAyIIVgB2G7xG0GP98zMAQ==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/big.js": {
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz",
@ -10330,6 +10408,22 @@
"url": "https://github.com/sponsors/fb55"
}
},
"node_modules/css-selector-parser": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/css-selector-parser/-/css-selector-parser-3.3.0.tgz",
"integrity": "sha512-Y2asgMGFqJKF4fq4xHDSlFYIkeVfRsm69lQC1q9kbEsH5XtnINTMrweLkjYMeaUgiXBy/uvKeO/a1JHTNnmB2g==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/mdevils"
},
{
"type": "patreon",
"url": "https://patreon.com/mdevils"
}
],
"license": "MIT"
},
"node_modules/css-tree": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz",
@ -11291,6 +11385,19 @@
"node": ">=8"
}
},
"node_modules/direction": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/direction/-/direction-2.0.1.tgz",
"integrity": "sha512-9S6m9Sukh1cZNknO1CWAr2QAWsbKLafQiyM5gZ7VgXHeuaoUwffKN4q6NC4A/Mf9iiPlOXQEKW/Mv/mh9/3YFA==",
"license": "MIT",
"bin": {
"direction": "cli.js"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/dns-packet": {
"version": "5.6.1",
"resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz",
@ -12812,6 +12919,38 @@
"node": ">= 0.4"
}
},
"node_modules/hast-util-embedded": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/hast-util-embedded/-/hast-util-embedded-3.0.0.tgz",
"integrity": "sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"hast-util-is-element": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-from-html": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/hast-util-from-html/-/hast-util-from-html-2.0.3.tgz",
"integrity": "sha512-CUSRHXyKjzHov8yKsQjGOElXy/3EKpyX56ELnkHH34vDVw1N1XSQ1ZcAvTyAPtGqLTuKP/uxM+aLkSPqF/EtMw==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"devlop": "^1.1.0",
"hast-util-from-parse5": "^8.0.0",
"parse5": "^7.0.0",
"vfile": "^6.0.0",
"vfile-message": "^4.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-from-parse5": {
"version": "8.0.3",
"resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-8.0.3.tgz",
@ -12832,6 +12971,62 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-has-property": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/hast-util-has-property/-/hast-util-has-property-3.0.0.tgz",
"integrity": "sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-is-body-ok-link": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/hast-util-is-body-ok-link/-/hast-util-is-body-ok-link-3.0.1.tgz",
"integrity": "sha512-0qpnzOBLztXHbHQenVB8uNuxTnm/QBFUOmdOSsEn7GnBtyY07+ENTWVFBAnXd/zEgd9/SUG3lRY7hSIBWRgGpQ==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-is-element": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-3.0.0.tgz",
"integrity": "sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-minify-whitespace": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/hast-util-minify-whitespace/-/hast-util-minify-whitespace-1.0.1.tgz",
"integrity": "sha512-L96fPOVpnclQE0xzdWb/D12VT5FabA7SnZOUMtL1DbXmYiHJMXZvFkIZfiMmTCNJHUeO2K9UYNXoVyfz+QHuOw==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"hast-util-embedded": "^3.0.0",
"hast-util-is-element": "^3.0.0",
"hast-util-whitespace": "^3.0.0",
"unist-util-is": "^6.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-parse-selector": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz",
@ -12845,6 +13040,23 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-phrasing": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/hast-util-phrasing/-/hast-util-phrasing-3.0.1.tgz",
"integrity": "sha512-6h60VfI3uBQUxHqTyMymMZnEbNl1XmEGtOxxKYL7stY2o601COo62AWAYBQR9lZbYXYSBoxag8UpPRXK+9fqSQ==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"hast-util-embedded": "^3.0.0",
"hast-util-has-property": "^3.0.0",
"hast-util-is-body-ok-link": "^3.0.0",
"hast-util-is-element": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-raw": {
"version": "9.1.0",
"resolved": "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-9.1.0.tgz",
@ -12870,6 +13082,33 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-select": {
"version": "6.0.4",
"resolved": "https://registry.npmjs.org/hast-util-select/-/hast-util-select-6.0.4.tgz",
"integrity": "sha512-RqGS1ZgI0MwxLaKLDxjprynNzINEkRHY2i8ln4DDjgv9ZhcYVIHN9rlpiYsqtFwrgpYU361SyWDQcGNIBVu3lw==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"@types/unist": "^3.0.0",
"bcp-47-match": "^2.0.0",
"comma-separated-tokens": "^2.0.0",
"css-selector-parser": "^3.0.0",
"devlop": "^1.0.0",
"direction": "^2.0.0",
"hast-util-has-property": "^3.0.0",
"hast-util-to-string": "^3.0.0",
"hast-util-whitespace": "^3.0.0",
"nth-check": "^2.0.0",
"property-information": "^7.0.0",
"space-separated-tokens": "^2.0.0",
"unist-util-visit": "^5.0.0",
"zwitch": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-estree": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/hast-util-to-estree/-/hast-util-to-estree-3.1.3.tgz",
@ -12898,6 +13137,29 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-html": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.5.tgz",
"integrity": "sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"@types/unist": "^3.0.0",
"ccount": "^2.0.0",
"comma-separated-tokens": "^2.0.0",
"hast-util-whitespace": "^3.0.0",
"html-void-elements": "^3.0.0",
"mdast-util-to-hast": "^13.0.0",
"property-information": "^7.0.0",
"space-separated-tokens": "^2.0.0",
"stringify-entities": "^4.0.0",
"zwitch": "^2.0.4"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-jsx-runtime": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz",
@ -12925,6 +13187,32 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-mdast": {
"version": "10.1.2",
"resolved": "https://registry.npmjs.org/hast-util-to-mdast/-/hast-util-to-mdast-10.1.2.tgz",
"integrity": "sha512-FiCRI7NmOvM4y+f5w32jPRzcxDIz+PUqDwEqn1A+1q2cdp3B8Gx7aVrXORdOKjMNDQsD1ogOr896+0jJHW1EFQ==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"@types/mdast": "^4.0.0",
"@ungap/structured-clone": "^1.0.0",
"hast-util-phrasing": "^3.0.0",
"hast-util-to-html": "^9.0.0",
"hast-util-to-text": "^4.0.0",
"hast-util-whitespace": "^3.0.0",
"mdast-util-phrasing": "^4.0.0",
"mdast-util-to-hast": "^13.0.0",
"mdast-util-to-string": "^4.0.0",
"rehype-minify-whitespace": "^6.0.0",
"trim-trailing-lines": "^2.0.0",
"unist-util-position": "^5.0.0",
"unist-util-visit": "^5.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-parse5": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-8.0.0.tgz",
@ -12954,6 +13242,35 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/hast-util-to-string": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-3.0.1.tgz",
"integrity": "sha512-XelQVTDWvqcl3axRfI0xSeoVKzyIFPwsAGSLIsKdJKQMXDYJS4WYrBNF/8J7RdhIcFI2BOHgAifggsvsxp/3+A==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-text": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-4.0.2.tgz",
"integrity": "sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"@types/unist": "^3.0.0",
"hast-util-is-element": "^3.0.0",
"unist-util-find-after": "^5.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-whitespace": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz",
@ -19478,6 +19795,15 @@
"react": "^16.8.0 || ^17 || ^18 || ^19"
}
},
"node_modules/react-icons": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/react-icons/-/react-icons-5.5.0.tgz",
"integrity": "sha512-MEFcXdkP3dLo8uumGI5xN3lDFNsRtrjbOEKDLD7yv76v4wpnEq2Lt2qeHaQOr34I/wPN3s3+N08WkQ+CW37Xiw==",
"license": "MIT",
"peerDependencies": {
"react": "*"
}
},
"node_modules/react-is": {
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
@ -19883,6 +20209,35 @@
"regjsparser": "bin/parser"
}
},
"node_modules/rehype-minify-whitespace": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/rehype-minify-whitespace/-/rehype-minify-whitespace-6.0.2.tgz",
"integrity": "sha512-Zk0pyQ06A3Lyxhe9vGtOtzz3Z0+qZ5+7icZ/PL/2x1SHPbKao5oB/g/rlc6BCTajqBb33JcOe71Ye1oFsuYbnw==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"hast-util-minify-whitespace": "^1.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/rehype-parse": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/rehype-parse/-/rehype-parse-9.0.1.tgz",
"integrity": "sha512-ksCzCD0Fgfh7trPDxr2rSylbwq9iYDkSn8TCDmEJ49ljEUBxDVCzCHv7QNzZOfODanX4+bWQ4WZqLCRWYLfhag==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"hast-util-from-html": "^2.0.0",
"unified": "^11.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/rehype-raw": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz",
@ -19913,6 +20268,23 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/rehype-remark": {
"version": "10.0.1",
"resolved": "https://registry.npmjs.org/rehype-remark/-/rehype-remark-10.0.1.tgz",
"integrity": "sha512-EmDndlb5NVwXGfUa4c9GPK+lXeItTilLhE6ADSaQuHr4JUlKw9MidzGzx4HpqZrNCt6vnHmEifXQiiA+CEnjYQ==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"@types/mdast": "^4.0.0",
"hast-util-to-mdast": "^10.0.0",
"unified": "^11.0.0",
"vfile": "^6.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/relateurl": {
"version": "0.2.7",
"resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz",
@ -21641,6 +22013,16 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/trim-trailing-lines": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/trim-trailing-lines/-/trim-trailing-lines-2.1.0.tgz",
"integrity": "sha512-5UR5Biq4VlVOtzqkm2AZlgvSlDJtME46uV0br0gENbwN4l5+mMKT4b9gJKqWtuL2zAIqajGJGuvbCbcAJUZqBg==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/trough": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz",
@ -21825,6 +22207,20 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/unist-util-find-after": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/unist-util-find-after/-/unist-util-find-after-5.0.0.tgz",
"integrity": "sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==",
"license": "MIT",
"dependencies": {
"@types/unist": "^3.0.0",
"unist-util-is": "^6.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/unist-util-is": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz",

2
docs/my-website/package.json
View file

@ -21,6 +21,8 @@
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "0.5.107",
"@mdx-js/react": "3.1.1",
"@signalwire/docusaurus-plugin-llms-txt": "2.0.0-alpha.7",
"@signalwire/docusaurus-theme-llms-txt": "1.0.0-alpha.9",
"clsx": "1.2.1",
"prism-react-renderer": "1.3.5",
"react": "18.3.1",

8
docs/my-website/sidebars.js
View file

@ -339,6 +339,13 @@ const sidebars = {
},
],
},
{
type: "category",
label: "Skills Gateway",
items: [
"skills_gateway",
],
},
],
},
{
@ -529,6 +536,7 @@ const sidebars = {
description: "Modify requests, responses, and more",
items: [
"proxy/call_hooks",
"proxy/agentic_loop_hook",
"proxy/rules",
]
},

196
enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py
View file

@ -3,6 +3,7 @@ Base class for sending emails to user after creating keys or invite links
"""
import html
import json
import os
from typing import List, Literal, Optional
@ -47,6 +48,15 @@ from litellm.secret_managers.main import get_secret_bool
from litellm.types.integrations.slack_alerting import LITELLM_LOGO_URL
def _parse_email_list(raw) -> List[str]:
"""Parse emails from a list or comma-separated string."""
if isinstance(raw, list):
return [e.strip() for e in raw if isinstance(e, str) and e.strip()]
elif isinstance(raw, str):
return [e.strip() for e in raw.split(",") if e.strip()]
return []
class BaseEmailLogger(CustomLogger):
DEFAULT_LITELLM_EMAIL = "notifications@alerts.litellm.ai"
DEFAULT_SUPPORT_EMAIL = "support@berri.ai"
@ -312,17 +322,22 @@ class BaseEmailLogger(CustomLogger):
)
pass
async def send_max_budget_alert_email(self, event: WebhookEvent):
async def send_max_budget_alert_email(
self,
event: WebhookEvent,
threshold_pct: Optional[int] = None,
recipient_emails: Optional[List[str]] = None,
):
"""
Send email to user when max budget alert threshold is reached
"""
email_params = await self._get_email_params(
email_event=EmailEvent.max_budget_alert,
user_id=event.user_id,
user_email=event.user_email,
event_message=event.event_message,
)
Send email to user when max budget alert threshold is reached.
Args:
event: The webhook event with spend/budget info
threshold_pct: Override percentage for multi-threshold alerts (e.g. 50, 75, 100).
When None, uses EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE (old behavior).
recipient_emails: Override recipient list for multi-threshold alerts.
When None, resolves single owner email via _get_email_params (old behavior).
"""
verbose_proxy_logger.debug(
f"send_max_budget_alert_email_event: {json.dumps(event.model_dump(exclude_none=True), indent=4, default=str)}"
)
@ -334,30 +349,67 @@ class BaseEmailLogger(CustomLogger):
)
# Calculate percentage and alert threshold
percentage = int(EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE * 100)
percentage = threshold_pct if threshold_pct is not None else int(
EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE * 100
)
threshold_fraction = percentage / 100.0
alert_threshold_str = (
f"${event.max_budget * EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE:.2f}"
f"${event.max_budget * threshold_fraction:.2f}"
if event.max_budget is not None
else "N/A"
)
email_html_content = MAX_BUDGET_ALERT_EMAIL_TEMPLATE.format(
email_logo_url=email_params.logo_url,
recipient_email=email_params.recipient_email,
percentage=percentage,
spend=spend_str,
max_budget=max_budget_str,
alert_threshold=alert_threshold_str,
base_url=email_params.base_url,
email_support_contact=email_params.support_contact,
)
await self.send_email(
from_email=self.DEFAULT_LITELLM_EMAIL,
to_email=[email_params.recipient_email],
subject=email_params.subject,
html_body=email_html_content,
)
pass
if recipient_emails:
# Multi-threshold path: batch send with generic key-based greeting
email_params = await self._get_email_params(
email_event=EmailEvent.max_budget_alert,
user_id=event.user_id,
user_email=event.user_email or recipient_emails[0],
event_message=event.event_message,
)
greeting = html.escape(
event.user_email or event.key_alias or event.token or ""
)
email_html_content = MAX_BUDGET_ALERT_EMAIL_TEMPLATE.format(
email_logo_url=email_params.logo_url,
recipient_email=greeting,
percentage=percentage,
spend=spend_str,
max_budget=max_budget_str,
alert_threshold=alert_threshold_str,
base_url=email_params.base_url,
email_support_contact=email_params.support_contact,
)
await self.send_email(
from_email=self.DEFAULT_LITELLM_EMAIL,
to_email=recipient_emails,
subject=email_params.subject,
html_body=email_html_content,
)
else:
# Old path: single recipient resolved from user_id/user_email
email_params = await self._get_email_params(
email_event=EmailEvent.max_budget_alert,
user_id=event.user_id,
user_email=event.user_email,
event_message=event.event_message,
)
email_html_content = MAX_BUDGET_ALERT_EMAIL_TEMPLATE.format(
email_logo_url=email_params.logo_url,
recipient_email=email_params.recipient_email,
percentage=percentage,
spend=spend_str,
max_budget=max_budget_str,
alert_threshold=alert_threshold_str,
base_url=email_params.base_url,
email_support_contact=email_params.support_contact,
)
await self.send_email(
from_email=self.DEFAULT_LITELLM_EMAIL,
to_email=[email_params.recipient_email],
subject=email_params.subject,
html_body=email_html_content,
)
async def budget_alerts(
self,
@ -469,6 +521,13 @@ class BaseEmailLogger(CustomLogger):
# For max_budget_alert, check if we've already sent an alert
if type == "max_budget_alert":
if user_info.max_budget is not None and user_info.spend is not None:
if user_info.max_budget_alert_emails:
# New path: multi-threshold alerts
await self._handle_multi_threshold_max_budget_alert(
user_info=user_info, _cache=_cache
)
return
alert_threshold = (
user_info.max_budget * EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE
)
@ -527,6 +586,87 @@ class BaseEmailLogger(CustomLogger):
)
return
async def _handle_multi_threshold_max_budget_alert(
self,
user_info: CallInfo,
_cache: DualCache,
):
"""
Loop over configured thresholds in max_budget_alert_emails,
check cache per threshold, and send to configured recipients.
"""
if not user_info.max_budget_alert_emails or user_info.max_budget is None:
return
for threshold_str, raw_emails in user_info.max_budget_alert_emails.items():
try:
threshold_pct = int(threshold_str)
except (ValueError, TypeError):
continue
threshold_amount = user_info.max_budget * (threshold_pct / 100.0)
if user_info.spend < threshold_amount:
continue
_id = user_info.token or user_info.user_id or "default_id"
_cache_key = (
f"email_budget_alerts:max_budget_alert:{threshold_pct}:{_id}"
)
result = await _cache.async_get_cache(key=_cache_key)
if result is not None:
continue
# Parse emails + auto-include owner
emails = _parse_email_list(raw_emails)
if user_info.user_email:
emails.append(user_info.user_email)
if not emails:
verbose_proxy_logger.warning(
"No recipients for %d%% threshold on key %s, skipping alert",
threshold_pct,
_id,
)
continue
recipient_emails = list(set(emails))
event_message = f"Max Budget Alert - {threshold_pct}% of Maximum Budget Reached"
webhook_event = WebhookEvent(
event="max_budget_alert",
event_message=event_message,
spend=user_info.spend,
max_budget=user_info.max_budget,
soft_budget=user_info.soft_budget,
token=user_info.token,
customer_id=user_info.customer_id,
user_id=user_info.user_id,
team_id=user_info.team_id,
team_alias=user_info.team_alias,
organization_id=user_info.organization_id,
user_email=user_info.user_email,
key_alias=user_info.key_alias,
projected_exceeded_date=user_info.projected_exceeded_date,
projected_spend=user_info.projected_spend,
event_group=user_info.event_group,
)
try:
await self.send_max_budget_alert_email(
webhook_event,
threshold_pct=threshold_pct,
recipient_emails=recipient_emails,
)
await _cache.async_set_cache(
key=_cache_key,
value="SENT",
ttl=EMAIL_BUDGET_ALERT_TTL,
)
except Exception as e:
verbose_proxy_logger.error(
f"Error sending multi-threshold max budget alert email for {threshold_pct}%: {e}",
exc_info=True,
)
async def _get_email_params(
self,
email_event: EmailEvent,

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.36"
version = "0.1.38"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -25,7 +25,7 @@ required-version = "==0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.36"
version = "0.1.38"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

7
litellm-proxy-extras/litellm_proxy_extras/_logging.py
View file

@ -23,7 +23,8 @@ class JsonFormatter(logging.Formatter):
def _is_json_enabled():
try:
import litellm
return getattr(litellm, 'json_logs', False)
return getattr(litellm, "json_logs", False)
except (ImportError, AttributeError):
return os.getenv("JSON_LOGS", "false").lower() == "true"
@ -35,6 +36,8 @@ if not logger.handlers:
if _is_json_enabled():
handler.setFormatter(JsonFormatter())
else:
handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
handler.setFormatter(
logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
)
logger.addHandler(handler)
logger.setLevel(logging.INFO)

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260401000000_add_budget_limits/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable: add budget_limits column to LiteLLM_VerificationToken
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "budget_limits" JSONB;
-- AlterTable: add budget_limits column to LiteLLM_TeamTable
ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN IF NOT EXISTS "budget_limits" JSONB;

9
litellm-proxy-extras/litellm_proxy_extras/migrations/20260401000000_add_team_member_model_scope/migration.sql Normal file
View file

@ -0,0 +1,9 @@
-- Add per-member model scope to LiteLLM_BudgetTable
-- allowed_models: empty array = inherit team models; non-empty = enforce member-level restriction
ALTER TABLE "LiteLLM_BudgetTable"
ADD COLUMN IF NOT EXISTS "allowed_models" TEXT[] DEFAULT ARRAY[]::TEXT[];
-- Add default_team_member_models to LiteLLM_TeamTable
-- Seeds allowed_models for newly added team members; empty = no per-member restriction
ALTER TABLE "LiteLLM_TeamTable"
ADD COLUMN IF NOT EXISTS "default_team_member_models" TEXT[] DEFAULT ARRAY[]::TEXT[];

6
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -17,8 +17,9 @@ model LiteLLM_BudgetTable {
tpm_limit BigInt?
rpm_limit BigInt?
model_max_budget Json?
budget_duration String?
budget_duration String?
budget_reset_at DateTime?
allowed_models String[] @default([]) // per-member model scope; empty = inherit team models
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
@ -140,6 +141,8 @@ model LiteLLM_TeamTable {
team_member_permissions String[] @default([])
access_group_ids String[] @default([])
policies String[] @default([])
default_team_member_models String[] @default([]) // default allowed_models for newly added team members; empty = no per-member restriction
budget_limits Json? // per-model budget limits for the team
model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases
allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
@ -401,6 +404,7 @@ model LiteLLM_VerificationToken {
rotation_interval String? // How often to rotate (e.g., "30d", "90d")
last_rotation_at DateTime? // When this key was last rotated
key_rotation_at DateTime? // When this key should next be rotated
budget_limits Json? // per-model budget limits for the key
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
litellm_project_table LiteLLM_ProjectTable? @relation(fields: [project_id], references: [project_id])

97
litellm-proxy-extras/litellm_proxy_extras/utils.py
View file

@ -4,8 +4,8 @@ import random
import re
import shutil
import subprocess
import tempfile
import time
from datetime import datetime
from pathlib import Path
from typing import Optional
@ -256,21 +256,11 @@ class ProxyExtrasDBManager:
if not database_url:
logger.error("DATABASE_URL not set")
return
# Prefer DIRECT_URL for schema introspection — pooler URLs (e.g. neon -pooler)
# do not support the extended query protocol required by prisma migrate diff.
diff_url = os.getenv("DIRECT_URL") or database_url
diff_dir = (
Path(migrations_dir)
/ "migrations"
/ f"{datetime.now().strftime('%Y%m%d%H%M%S')}_baseline_diff"
)
try:
diff_dir.mkdir(parents=True, exist_ok=True)
except Exception as e:
if "Permission denied" in str(e):
logger.warning(
f"Permission denied - {e}\nunable to baseline db. Set LITELLM_MIGRATION_DIR environment variable to a writable directory to enable migrations."
)
return
raise e
diff_dir = Path(tempfile.mkdtemp(prefix="litellm_migration_diff_"))
diff_sql_path = diff_dir / "migration.sql"
# 1. Generate migration SQL for the diff between DB and schema
@ -283,7 +273,7 @@ class ProxyExtrasDBManager:
"migrate",
"diff",
"--from-url",
database_url,
diff_url,
"--to-schema-datamodel",
schema_path,
"--script",
@ -300,7 +290,40 @@ class ProxyExtrasDBManager:
# check if the migration was created
if not diff_sql_path.exists():
logger.warning("Migration diff was not created")
logger.warning(
"Migration diff was not created (prisma migrate diff failed — "
"likely a pooler URL). Falling back to direct SQL execution of "
"each migration file."
)
# Fall back: run each migration SQL file directly via prisma db execute.
# This works with pooler URLs (no schema introspection needed) and is
# safe to re-run because migrations use IF NOT EXISTS / IF EXISTS guards.
migration_files = sorted(Path(migrations_dir).glob("*/migration.sql"))
for mig_file in migration_files:
try:
subprocess.run(
[
_get_prisma_command(),
"db",
"execute",
"--file",
str(mig_file),
"--schema",
schema_path,
],
timeout=60,
check=True,
capture_output=True,
text=True,
env=_get_prisma_env(),
)
logger.info(f"Applied migration: {mig_file.parent.name}")
except subprocess.CalledProcessError as e:
logger.warning(
f"Failed to apply migration {mig_file.parent.name}: {e.stderr}"
)
except subprocess.TimeoutExpired:
logger.warning(f"Migration {mig_file.parent.name} timed out.")
return
logger.info(f"Migration diff created at {diff_sql_path}")
@ -395,6 +418,14 @@ class ProxyExtrasDBManager:
logger.info("prisma migrate deploy completed")
# Skip sanity check when deploy reports no pending migrations —
# DB already matches schema, no drift to correct.
if "No pending migrations to apply" in result.stdout:
logger.info(
"No pending migrations — skipping post-migration sanity check"
)
return True
# Run sanity check to ensure DB matches schema
logger.info("Running post-migration sanity check...")
ProxyExtrasDBManager._resolve_all_migrations(
@ -419,7 +450,10 @@ class ProxyExtrasDBManager:
ProxyExtrasDBManager._roll_back_migration(
failed_migration
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as rollback_err:
except (
subprocess.CalledProcessError,
subprocess.TimeoutExpired,
) as rollback_err:
logger.warning(
f"Failed to roll back migration {failed_migration}: {rollback_err}. "
f"It may already be in a rolled-back state."
@ -431,10 +465,19 @@ class ProxyExtrasDBManager:
logger.info(
f"✅ Migration {failed_migration} resolved, retrying to apply remaining migrations"
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as resolve_err:
except (
subprocess.CalledProcessError,
subprocess.TimeoutExpired,
) as resolve_err:
logger.warning(
f"Failed to resolve migration {failed_migration}: {resolve_err}"
)
# Apply any schema drift not covered by the marked-as-applied migration
ProxyExtrasDBManager._resolve_all_migrations(
migrations_dir,
schema_path,
mark_all_applied=False,
)
else:
logger.info(
f"Found failed migration: {failed_migration}, marking as rolled back"
@ -531,7 +574,10 @@ class ProxyExtrasDBManager:
ProxyExtrasDBManager._roll_back_migration(
migration_name
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as rollback_err:
except (
subprocess.CalledProcessError,
subprocess.TimeoutExpired,
) as rollback_err:
logger.warning(
f"Failed to roll back migration {migration_name}: {rollback_err}. "
f"It may already be in a rolled-back state."
@ -548,10 +594,19 @@ class ProxyExtrasDBManager:
f"✅ Migration {migration_name} resolved, "
f"retrying to apply remaining migrations"
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as resolve_err:
except (
subprocess.CalledProcessError,
subprocess.TimeoutExpired,
) as resolve_err:
logger.warning(
f"Failed to resolve migration {migration_name}: {resolve_err}"
)
# Apply any schema drift not covered by the marked-as-applied migration
ProxyExtrasDBManager._resolve_all_migrations(
migrations_dir,
schema_path,
mark_all_applied=False,
)
else:
# Unknown P3018 error - log and re-raise for safety
logger.warning(

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.66"
version = "0.4.67"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -25,7 +25,7 @@ required-version = "==0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.66"
version = "0.4.67"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

124
litellm/__init__.py
View file

@ -148,6 +148,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"vantage",
"posthog",
"levo",
"compression_interception",
]
cold_storage_custom_logger: Optional[_custom_logger_compatible_callbacks_literal] = None
logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
@ -168,12 +169,12 @@ prometheus_latency_buckets: Optional[List[float]] = None
require_auth_for_metrics_endpoint: Optional[bool] = False
argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
gcs_pub_sub_use_v1: Optional[
bool
] = False # if you want to use v1 gcs pubsub logged payload
generic_api_use_v1: Optional[
bool
] = False # if you want to use v1 generic api logged payload
gcs_pub_sub_use_v1: Optional[bool] = (
False # if you want to use v1 gcs pubsub logged payload
)
generic_api_use_v1: Optional[bool] = (
False # if you want to use v1 generic api logged payload
)
argilla_transformation_object: Optional[Dict[str, Any]] = None
_async_input_callback: List[
Union[str, Callable, "CustomLogger"]
@ -193,26 +194,26 @@ _async_failure_callback: List[
pre_call_rules: List[Callable] = []
post_call_rules: List[Callable] = []
turn_off_message_logging: Optional[bool] = False
standard_logging_payload_excluded_fields: Optional[
List[str]
] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it
standard_logging_payload_excluded_fields: Optional[List[str]] = (
None # Fields to exclude from StandardLoggingPayload before callbacks receive it
)
log_raw_request_response: bool = False
redact_messages_in_exceptions: Optional[bool] = False
redact_user_api_key_info: Optional[bool] = False
filter_invalid_headers: Optional[bool] = False
add_user_information_to_llm_headers: Optional[
bool
] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
add_user_information_to_llm_headers: Optional[bool] = (
None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
)
store_audit_logs = False # Enterprise feature, allow users to see audit logs
skip_system_message_in_guardrail: bool = False
### end of callbacks #############
email: Optional[
str
] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
token: Optional[
str
] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
email: Optional[str] = (
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
token: Optional[str] = (
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
telemetry = True
max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults
drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False))
@ -274,9 +275,11 @@ use_client: bool = False
ssl_verify: Union[str, bool] = True
ssl_security_level: Optional[str] = None
ssl_certificate: Optional[str] = None
ssl_ecdh_curve: Optional[
str
] = None # Set to 'X25519' to disable PQC and improve performance
user_url_validation: bool = True
user_url_allowed_hosts: List[str] = []
ssl_ecdh_curve: Optional[str] = (
None # Set to 'X25519' to disable PQC and improve performance
)
disable_streaming_logging: bool = False
disable_token_counter: bool = False
disable_add_transform_inline_image_block: bool = False
@ -330,20 +333,24 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
enable_caching_on_provider_specific_optional_params: bool = (
False # feature-flag for caching on optional params - e.g. 'top_k'
)
caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
cache: Optional[
"Cache"
] = None # cache object <- use this - https://docs.litellm.ai/docs/caching
caching: bool = (
False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
caching_with_models: bool = (
False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
cache: Optional["Cache"] = (
None # cache object <- use this - https://docs.litellm.ai/docs/caching
)
default_in_memory_ttl: Optional[float] = None
default_redis_ttl: Optional[float] = None
default_redis_batch_cache_expiry: Optional[float] = None
model_alias_map: Dict[str, str] = {}
model_group_settings: Optional["ModelGroupSettings"] = None
max_budget: float = 0.0 # set the max budget across all providers
budget_duration: Optional[
str
] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
budget_duration: Optional[str] = (
None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
)
default_soft_budget: float = (
DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0
)
@ -352,7 +359,9 @@ forward_traceparent_to_llm_provider: bool = False
_current_cost = 0.0 # private variable, used if max budget is set
error_logs: Dict = {}
add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt
add_function_to_prompt: bool = (
False # if function calling not supported by api, append function call details to system prompt
)
client_session: Optional[httpx.Client] = None
aclient_session: Optional[httpx.AsyncClient] = None
model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks'
@ -376,6 +385,7 @@ datadog_params: Optional[Union[DatadogInitParams, Dict]] = None
aws_sqs_callback_params: Optional[Dict] = None
generic_logger_headers: Optional[Dict] = None
default_key_generate_params: Optional[Dict] = None
default_key_max_budget_alert_emails: Optional[Dict[str, list]] = None
upperbound_key_generate_params: Optional[LiteLLM_UpperboundKeyGenerateParams] = None
key_generation_settings: Optional["StandardKeyGenerationConfig"] = None
default_internal_user_params: Optional[Dict] = None
@ -399,7 +409,9 @@ prometheus_emit_stream_label: bool = False
disable_add_prefix_to_prompt: bool = (
False # used by anthropic, to disable adding prefix to prompt
)
disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
disable_copilot_system_to_assistant: bool = (
False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
)
public_mcp_servers: Optional[List[str]] = None
public_model_groups: Optional[List[str]] = None
public_agent_groups: Optional[List[str]] = None
@ -408,9 +420,9 @@ public_agent_groups: Optional[List[str]] = None
# Old format: { "displayName": "url" } (for backward compatibility)
public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
#### REQUEST PRIORITIZATION #######
priority_reservation: Optional[
Dict[str, Union[float, "PriorityReservationDict"]]
] = None
priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = (
None
)
# priority_reservation_settings is lazy-loaded via __getattr__
# Only declare for type checking - at runtime __getattr__ handles it
if TYPE_CHECKING:
@ -418,13 +430,17 @@ if TYPE_CHECKING:
######## Networking Settings ########
use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead.
use_aiohttp_transport: bool = (
True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead.
)
aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings
disable_aiohttp_transport: bool = False # Set this to true to use httpx instead
disable_aiohttp_trust_env: bool = (
False # When False, aiohttp will respect HTTP(S)_PROXY env vars
)
force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
force_ipv4: bool = (
False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
)
network_mock: bool = False # When True, use mock transport — no real network calls
####### STOP SEQUENCE LIMIT #######
@ -439,13 +455,13 @@ context_window_fallbacks: Optional[List] = None
content_policy_fallbacks: Optional[List] = None
allowed_fails: int = 3
allow_dynamic_callback_disabling: bool = True
num_retries_per_request: Optional[
int
] = None # for the request overall (incl. fallbacks + model retries)
num_retries_per_request: Optional[int] = (
None # for the request overall (incl. fallbacks + model retries)
)
####### SECRET MANAGERS #####################
secret_manager_client: Optional[
Any
] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
secret_manager_client: Optional[Any] = (
None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
)
_google_kms_resource_name: Optional[str] = None
_key_management_system: Optional["KeyManagementSystem"] = None
# Note: KeyManagementSettings must be eagerly imported because _key_management_settings
@ -458,12 +474,12 @@ output_parse_pii: bool = False
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map
model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[
str, float
] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
cost_margin_config: Dict[
str, Union[float, Dict[str, float]]
] = {} # Provider-specific or global cost margins. Examples:
cost_discount_config: Dict[str, float] = (
{}
) # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
cost_margin_config: Dict[str, Union[float, Dict[str, float]]] = (
{}
) # Provider-specific or global cost margins. Examples:
# Percentage: {"openai": 0.10} = 10% margin
# Fixed: {"openai": {"fixed_amount": 0.001}} = $0.001 per request
# Global: {"global": 0.05} = 5% global margin on all providers
@ -1313,12 +1329,12 @@ from . import rag
from .types.llms.custom_llm import CustomLLMItem
custom_provider_map: List[CustomLLMItem] = []
_custom_providers: List[
str
] = [] # internal helper util, used to track names of custom providers
disable_hf_tokenizer_download: Optional[
bool
] = None # disable huggingface tokenizer download. Defaults to openai clk100
_custom_providers: List[str] = (
[]
) # internal helper util, used to track names of custom providers
disable_hf_tokenizer_download: Optional[bool] = (
None # disable huggingface tokenizer download. Defaults to openai clk100
)
global_disable_no_log_param: bool = False
### CLI UTILITIES ###

1
litellm/_lazy_imports.py
View file

@ -14,6 +14,7 @@ How it works:
This makes importing litellm much faster because we don't load heavy dependencies
until they're actually needed.
"""
import importlib
import sys
from typing import Any, Optional, cast, Callable

6
litellm/a2a_protocol/main.py
View file

@ -120,9 +120,9 @@ def _get_a2a_model_info(a2a_client: Any, kwargs: Dict[str, Any]) -> str:
litellm_logging_obj.model = model
litellm_logging_obj.custom_llm_provider = custom_llm_provider
litellm_logging_obj.model_call_details["model"] = model
litellm_logging_obj.model_call_details[
"custom_llm_provider"
] = custom_llm_provider
litellm_logging_obj.model_call_details["custom_llm_provider"] = (
custom_llm_provider
)
return agent_name

4
litellm/a2a_protocol/providers/bedrock_agentcore/handler.py
View file

@ -99,9 +99,7 @@ class BedrockAgentCoreA2AHandler:
)
)
verbose_logger.info(
f"BedrockAgentCore A2A: Sending streaming request to {url}"
)
verbose_logger.info(f"BedrockAgentCore A2A: Sending streaming request to {url}")
client = get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),

6
litellm/a2a_protocol/streaming_iterator.py
View file

@ -168,9 +168,9 @@ class A2AStreamingIterator:
result: Dict[str, Any] = {
"id": getattr(self.request, "id", "unknown"),
"jsonrpc": "2.0",
"usage": usage.model_dump()
if hasattr(usage, "model_dump")
else dict(usage),
"usage": (
usage.model_dump() if hasattr(usage, "model_dump") else dict(usage)
),
}
# Add final chunk result if available

1
litellm/anthropic_interface/__init__.py
View file

@ -1,6 +1,7 @@
"""
Anthropic module for LiteLLM
"""
from .messages import acreate, create
__all__ = ["acreate", "create"]

4
litellm/anthropic_interface/messages/__init__.py
View file

@ -38,7 +38,7 @@ async def acreate(
top_k: Optional[int] = None,
top_p: Optional[float] = None,
container: Optional[Dict] = None,
**kwargs
**kwargs,
) -> Union[AnthropicMessagesResponse, AsyncIterator]:
"""
Async wrapper for Anthropic's messages API
@ -97,7 +97,7 @@ def create(
top_k: Optional[int] = None,
top_p: Optional[float] = None,
container: Optional[Dict] = None,
**kwargs
**kwargs,
) -> Union[
AnthropicMessagesResponse,
AsyncIterator[Any],

10
litellm/caching/caching_handler.py
View file

@ -78,7 +78,9 @@ class CachingHandlerResponse(BaseModel):
cached_result: Optional[Any] = None
final_embedding_cached_response: Optional[EmbeddingResponse] = None
embedding_all_elements_cache_hit: bool = False # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call
embedding_all_elements_cache_hit: bool = (
False # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call
)
in_memory_cache_obj = InMemoryCache()
@ -1014,9 +1016,9 @@ class LLMCachingHandler:
}
if litellm.cache is not None:
litellm_params[
"preset_cache_key"
] = litellm.cache._get_preset_cache_key_from_kwargs(**kwargs)
litellm_params["preset_cache_key"] = (
litellm.cache._get_preset_cache_key_from_kwargs(**kwargs)
)
else:
litellm_params["preset_cache_key"] = None

1
litellm/caching/gcs_cache.py
View file

@ -1,6 +1,7 @@
"""GCS Cache implementation
Supports syncing responses to Google Cloud Storage Buckets using HTTP requests.
"""
import json
import asyncio
from typing import Optional

4
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -142,9 +142,7 @@ class ResponsesToCompletionBridgeHandler:
custom_llm_provider=custom_llm_provider,
)
def completion(
self, *args, **kwargs
) -> Union[
def completion(self, *args, **kwargs) -> Union[
Coroutine[Any, Any, Union["ModelResponse", "CustomStreamWrapper"]],
"ModelResponse",
"CustomStreamWrapper",

36
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -300,10 +300,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if key in ("max_tokens", "max_completion_tokens"):
responses_api_request["max_output_tokens"] = value
elif key == "tools" and value is not None:
responses_api_request[
"tools"
] = self._convert_tools_to_responses_format(
cast(List[Dict[str, Any]], value)
responses_api_request["tools"] = (
self._convert_tools_to_responses_format(
cast(List[Dict[str, Any]], value)
)
)
elif key == "response_format":
text_format = self._transform_response_format_to_text_format(value)
@ -506,9 +506,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
annotations=annotations,
reasoning_items=cast(
Optional[List[ChatCompletionReasoningItem]],
[pending_reasoning_item]
if pending_reasoning_item is not None
else None,
(
[pending_reasoning_item]
if pending_reasoning_item is not None
else None
),
),
)
@ -566,9 +568,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
reasoning_content=reasoning_content,
reasoning_items=cast(
Optional[List[ChatCompletionReasoningItem]],
[pending_reasoning_item]
if pending_reasoning_item is not None
else None,
(
[pending_reasoning_item]
if pending_reasoning_item is not None
else None
),
),
)
choices.append(
@ -1154,9 +1158,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
)
if provider_specific_fields:
function_chunk[
"provider_specific_fields"
] = provider_specific_fields
function_chunk["provider_specific_fields"] = (
provider_specific_fields
)
tool_call_index = parsed_chunk.get("output_index", 0)
tool_call_chunk = ChatCompletionToolCallChunk(
@ -1229,9 +1233,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
# Add provider_specific_fields to function if present
if provider_specific_fields:
function_chunk[
"provider_specific_fields"
] = provider_specific_fields
function_chunk["provider_specific_fields"] = (
provider_specific_fields
)
tool_call_index = parsed_chunk.get("output_index", 0)
tool_call_chunk = ChatCompletionToolCallChunk(

420
litellm/compression/compress.py
View file

@ -1,9 +1,9 @@
"""
Main compress() function — orchestrates BM25/embedding scoring, message stubbing,
and retrieval tool injection.
Main compress() function — normalizes input messages, orchestrates BM25/embedding
scoring, message stubbing, and retrieval tool injection.
"""
from typing import Any, Dict, List, Optional, Set, Union, cast
from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
from litellm.caching.dual_cache import DualCache
from litellm.compression.message_stubbing import (
@ -15,27 +15,196 @@ from litellm.compression.retrieval_tool import build_retrieval_tool
from litellm.compression.scoring.bm25 import bm25_score_messages
from litellm.litellm_core_utils.token_counter import token_counter
from litellm.types.compression import CompressedResult
from litellm.types.utils import AllMessageValues, Message
from litellm.types.utils import CallTypes
# CallTypes that produce Anthropic-shaped messages (structured content blocks).
# Everything else is treated as OpenAI chat-completions shape.
_ANTHROPIC_CALL_TYPES = frozenset({CallTypes.anthropic_messages.value})
# CallTypes that are valid targets for compression. Compression operates on
# message-shaped inputs, so we only accept call types whose payload is a list
# of role/content messages.
_SUPPORTED_CALL_TYPES = frozenset(
{
CallTypes.completion.value,
CallTypes.acompletion.value,
CallTypes.anthropic_messages.value,
}
)
def _normalize_call_type(call_type: Union[CallTypes, str]) -> str:
"""Return the string value for a ``CallTypes`` enum or a raw string."""
if isinstance(call_type, CallTypes):
return call_type.value
return call_type
def _is_anthropic_call_type(call_type: str) -> bool:
return call_type in _ANTHROPIC_CALL_TYPES
def _build_retrieval_tools(keys: List[str], call_type: str) -> List[dict]:
"""
Build retrieval tool definitions in the target request schema.
- Chat-completions call types: keep OpenAI function-tool schema.
- Anthropic messages call type: remap to Anthropic's custom tool schema.
"""
if not keys:
return []
openai_tools = [build_retrieval_tool(keys)]
if not _is_anthropic_call_type(call_type):
return openai_tools
# Lazy import to avoid introducing provider transformation imports during
# module import for non-Anthropic call paths.
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
anthropic_tools, _mcp_servers = AnthropicConfig()._map_tools(openai_tools)
return cast(List[dict], anthropic_tools)
def _content_to_text(content: Any) -> str:
"""
Convert OpenAI/Anthropic message content blocks to plain text.
Text extraction policy:
- Include text-bearing fields only (`text` blocks + string values).
- For `tool_result`, expand into nested `content` items.
- Ignore non-textual blocks (images/documents/tool metadata/thinking metadata).
Implemented iteratively (stack-based) to avoid unbounded recursion.
"""
parts: List[str] = []
stack: List[Any] = [content]
while stack:
item = stack.pop()
if isinstance(item, str):
parts.append(item)
elif isinstance(item, list):
# Push list items in reverse order so they are processed left-to-right.
for element in reversed(item):
stack.append(element)
elif isinstance(item, dict):
item_type = item.get("type")
if item_type == "text":
parts.append(str(item.get("text", "")))
elif item_type == "tool_result":
stack.append(item.get("content", ""))
return " ".join(parts)
def _normalize_messages_for_compression(
messages: List[dict],
call_type: str,
) -> Tuple[List[dict], List[dict]]:
"""
Normalize each original message to a text-surrogate content for scoring.
Returns:
(normalized_messages, original_messages_copy)
"""
if call_type not in _SUPPORTED_CALL_TYPES:
raise ValueError(
f"Unsupported call_type={call_type!r} for compression. "
f"Expected one of: {sorted(_SUPPORTED_CALL_TYPES)}."
)
original_messages: List[Dict[str, Any]] = [dict(m) for m in messages]
normalized_messages: List[dict] = []
for msg in original_messages:
normalized_messages.append(
{
**msg,
"content": _content_to_text(msg.get("content", "")),
}
)
return normalized_messages, original_messages
def _extract_last_user_message(messages: List[dict]) -> str:
"""Return the text content of the last user message."""
for msg in reversed(messages):
if msg.get("role") == "user":
content = msg.get("content", "")
if isinstance(content, str):
return content
if isinstance(content, list):
parts = []
for part in content:
if isinstance(part, dict) and part.get("type") == "text":
parts.append(part.get("text", ""))
elif isinstance(part, str):
parts.append(part)
return " ".join(parts)
return _content_to_text(msg.get("content", ""))
return ""
def _extract_tool_use_ids(content: Any) -> List[str]:
if not isinstance(content, list):
return []
tool_use_ids: List[str] = []
for part in content:
if not isinstance(part, dict):
continue
if part.get("type") != "tool_use":
continue
tool_use_id = part.get("id")
if isinstance(tool_use_id, str) and tool_use_id:
tool_use_ids.append(tool_use_id)
return tool_use_ids
def _extract_tool_result_ids(content: Any) -> Set[str]:
if not isinstance(content, list):
return set()
tool_result_ids: Set[str] = set()
for part in content:
if not isinstance(part, dict):
continue
if part.get("type") != "tool_result":
continue
tool_use_id = part.get("tool_use_id")
if isinstance(tool_use_id, str) and tool_use_id:
tool_result_ids.add(tool_use_id)
return tool_result_ids
def _extract_anthropic_tool_exchange_spans(
messages: List[dict],
) -> Tuple[List[Set[int]], Optional[str]]:
"""
Return atomic 2-message spans for Anthropic tool exchanges.
Each assistant message containing `tool_use` must be immediately followed by a
user message containing matching `tool_result` blocks for all tool_use ids.
"""
spans: List[Set[int]] = []
i = 0
while i < len(messages):
current = messages[i]
if current.get("role") != "assistant":
i += 1
continue
tool_use_ids = _extract_tool_use_ids(current.get("content"))
if not tool_use_ids:
i += 1
continue
if i + 1 >= len(messages):
return [], "invalid_anthropic_tool_sequence"
next_msg = messages[i + 1]
if next_msg.get("role") != "user":
return [], "invalid_anthropic_tool_sequence"
tool_result_ids = _extract_tool_result_ids(next_msg.get("content"))
if not tool_result_ids:
return [], "invalid_anthropic_tool_sequence"
for tool_use_id in tool_use_ids:
if tool_use_id not in tool_result_ids:
return [], "invalid_anthropic_tool_sequence"
spans.append({i, i + 1})
i += 2
return spans, None
def _get_protected_indices(messages: List[dict]) -> List[int]:
"""
Return indices of messages that must never be compressed:
@ -87,9 +256,98 @@ def _combine_scores(
return [bm25_weight * b + emb_weight * e for b, e in zip(norm_bm25, norm_emb)]
def _select_kept_indices_for_budget(
normalized_messages: List[dict],
original_messages: List[dict],
combined_scores: List[float],
compression_target: int,
model: str,
initial_kept_indices: Set[int],
tool_exchange_spans: List[Set[int]],
) -> Tuple[Set[int], Dict[int, dict]]:
kept_indices = set(initial_kept_indices)
current_tokens = 0
for i in kept_indices:
current_tokens += token_counter(
model=model,
text=cast(str, normalized_messages[i].get("content", "") or ""),
)
# Fill token budget from highest-scoring units.
# A unit is either:
# 1) a single message index, or
# 2) an Anthropic tool-exchange span that must be kept/dropped atomically.
truncated_overrides: Dict[int, dict] = {} # idx -> truncated message dict
span_id_by_index: Dict[int, int] = {}
for span_id, span in enumerate(tool_exchange_spans):
for idx in span:
span_id_by_index[idx] = span_id
# Build single-message candidate units (non-span messages).
candidate_units: List[Tuple[float, Tuple[int, ...], bool]] = []
for idx in range(len(normalized_messages)):
if idx in span_id_by_index or idx in kept_indices:
continue
candidate_units.append((combined_scores[idx], (idx,), True))
# Build span candidate units (atomic keep/drop for tool exchanges).
for span in tool_exchange_spans:
span_indices = tuple(sorted(span))
if any(idx in kept_indices for idx in span_indices):
continue
span_score = max(combined_scores[idx] for idx in span_indices)
candidate_units.append((span_score, span_indices, False))
# Sort by descending relevance score.
candidate_units.sort(key=lambda item: item[0], reverse=True)
for _score, indices, can_truncate in candidate_units:
if any(idx in kept_indices for idx in indices):
continue
msg_tokens = 0
for idx in indices:
msg_tokens += token_counter(
model=model,
text=cast(str, normalized_messages[idx].get("content", "") or ""),
)
remaining = compression_target - current_tokens
if remaining <= 0:
break # budget exhausted
if current_tokens + msg_tokens <= compression_target:
# Fits entirely
kept_indices.update(indices)
current_tokens += msg_tokens
elif can_truncate and len(indices) == 1 and remaining >= 100:
# Too large to fit whole single message, but we have budget — truncate it.
idx = indices[0]
truncated = truncate_message(original_messages[idx], remaining)
truncated_tokens = token_counter(
model=model,
text=truncated.get("content", "") or "",
)
truncated_overrides[idx] = truncated
kept_indices.add(idx)
current_tokens += truncated_tokens
return kept_indices, truncated_overrides
def _get_dropped_tool_span_indices(
kept_indices: Set[int], tool_exchange_spans: List[Set[int]]
) -> Set[int]:
dropped_tool_span_indices: Set[int] = set()
for span in tool_exchange_spans:
if not any(idx in kept_indices for idx in span):
dropped_tool_span_indices.update(span)
return dropped_tool_span_indices
def compress(
messages: List[dict],
model: str,
call_type: Union[CallTypes, str] = CallTypes.completion,
compression_trigger: int = 200_000,
compression_target: Optional[int] = None,
embedding_model: Optional[str] = None,
@ -108,6 +366,12 @@ def compress(
Parameters:
messages: The conversation messages to (potentially) compress.
model: The LLM model name — used for token counting.
call_type: The LiteLLM call type whose message schema these messages
follow. Supported values:
- ``CallTypes.completion`` / ``CallTypes.acompletion`` — OpenAI
chat-completions shape (default)
- ``CallTypes.anthropic_messages`` — Anthropic Messages shape
(structured content blocks + atomic tool exchanges)
compression_trigger: Only compress if input exceeds this token count.
compression_target: Target token count after compression.
Defaults to ``compression_trigger // 2``.
@ -122,29 +386,37 @@ def compress(
A ``CompressedResult`` dict containing compressed messages, token
counts, a cache of original content, and the retrieval tool definition.
"""
call_type_str = _normalize_call_type(call_type)
normalized_messages, original_messages = _normalize_messages_for_compression(
messages=messages,
call_type=call_type_str,
)
if compression_target is None:
compression_target = compression_trigger * 7 // 10
original_tokens = token_counter(
model=model, messages=cast(List[Union[AllMessageValues, Message]], messages)
model=model,
messages=cast(List[Any], original_messages),
)
# Pass through if below trigger
if original_tokens <= compression_trigger:
return CompressedResult(
messages=messages,
messages=original_messages,
original_tokens=original_tokens,
compressed_tokens=original_tokens,
compression_ratio=0.0,
cache={},
tools=[],
compression_skipped_reason="below_trigger",
)
# Extract query for relevance scoring
query = _extract_last_user_message(messages)
query = _extract_last_user_message(normalized_messages)
# Score each message
bm25_scores = bm25_score_messages(query, messages)
bm25_scores = bm25_score_messages(query, normalized_messages)
if embedding_model:
from litellm.compression.scoring.embedding_scorer import (
@ -153,7 +425,7 @@ def compress(
emb_scores = embedding_score_messages(
query,
messages,
normalized_messages,
model=embedding_model,
cache=compression_cache,
embedding_model_params=embedding_model_params,
@ -162,94 +434,80 @@ def compress(
else:
combined_scores = bm25_scores
# Sort message indices by score descending
ranked_indices = sorted(
range(len(messages)),
key=lambda i: combined_scores[i],
reverse=True,
)
# Protected messages are never compressed
protected_indices = _get_protected_indices(messages)
protected_indices = _get_protected_indices(normalized_messages)
kept_indices: Set[int] = set(protected_indices)
# Count tokens for protected messages
current_tokens = 0
for i in kept_indices:
current_tokens += token_counter(
model=model, text=messages[i].get("content", "") or ""
tool_exchange_spans: List[Set[int]] = []
if _is_anthropic_call_type(call_type_str):
tool_exchange_spans, tool_sequence_error = (
_extract_anthropic_tool_exchange_spans(original_messages)
)
# Fill token budget from highest-scoring messages.
# For each candidate (ranked by relevance):
# - If it fits entirely → keep it as-is.
# - If it doesn't fit but there's meaningful remaining budget → truncate it
# to fill as much of the budget as possible.
# - Otherwise → stub it (pointer only, content goes to cache).
# Multiple messages may be truncated so we preserve partial content from
# several high-scoring messages rather than fully stubbing all but one.
truncated_overrides: Dict[int, dict] = {} # idx -> truncated message dict
for idx in ranked_indices:
if idx in kept_indices:
continue
msg_content = messages[idx].get("content", "") or ""
msg_tokens = token_counter(model=model, text=msg_content)
remaining = compression_target - current_tokens
if remaining <= 0:
break # budget exhausted
if current_tokens + msg_tokens <= compression_target:
# Fits entirely
kept_indices.add(idx)
current_tokens += msg_tokens
elif remaining >= 100:
# Too large to fit whole, but we have budget — truncate it.
truncated = truncate_message(messages[idx], remaining)
truncated_tokens = token_counter(
model=model,
text=truncated.get("content", "") or "",
if tool_sequence_error is not None:
return CompressedResult(
messages=original_messages,
original_tokens=original_tokens,
compressed_tokens=original_tokens,
compression_ratio=0.0,
cache={},
tools=[],
compression_skipped_reason=tool_sequence_error,
)
truncated_overrides[idx] = truncated
kept_indices.add(idx)
current_tokens += truncated_tokens
for span in tool_exchange_spans:
# If any message in the span is protected, keep the whole span.
if any(idx in kept_indices for idx in span):
kept_indices.update(span)
kept_indices, truncated_overrides = _select_kept_indices_for_budget(
normalized_messages=normalized_messages,
original_messages=original_messages,
combined_scores=combined_scores,
compression_target=compression_target,
model=model,
initial_kept_indices=kept_indices,
tool_exchange_spans=tool_exchange_spans,
)
# Build compressed messages and cache
compressed_messages: List[dict] = []
cache: Dict[str, str] = {}
used_keys: Set[str] = set()
dropped_tool_span_indices = _get_dropped_tool_span_indices(
kept_indices=kept_indices, tool_exchange_spans=tool_exchange_spans
)
for i, msg in enumerate(messages):
for i, msg in enumerate(original_messages):
if i in dropped_tool_span_indices:
continue
if i in kept_indices:
# Use the truncated version if we made one, otherwise the original
compressed_messages.append(truncated_overrides.get(i, msg))
else:
key = extract_key(msg, fallback_index=i, used_keys=used_keys)
content = msg.get("content", "")
if isinstance(content, list):
content = " ".join(
p.get("text", "") if isinstance(p, dict) else str(p)
for p in content
)
key = extract_key(
normalized_messages[i], fallback_index=i, used_keys=used_keys
)
content = _content_to_text(msg.get("content", ""))
cache[key] = content
compressed_messages.append(stub_message(msg, key))
# Build retrieval tool
tools = [build_retrieval_tool(list(cache.keys()))] if cache else []
# Build retrieval tool in the target request schema
tools = _build_retrieval_tools(list(cache.keys()), call_type=call_type_str)
compressed_tokens = token_counter(
model=model,
messages=cast(List[Union[AllMessageValues, Message]], compressed_messages),
messages=cast(List[Any], compressed_messages),
)
return CompressedResult(
messages=compressed_messages,
original_tokens=original_tokens,
compressed_tokens=compressed_tokens,
compression_ratio=round(1 - (compressed_tokens / original_tokens), 4)
if original_tokens > 0
else 0.0,
compression_ratio=(
round(1 - (compressed_tokens / original_tokens), 4)
if original_tokens > 0
else 0.0
),
cache=cache,
tools=tools,
)

19
litellm/constants.py
View file

@ -1360,6 +1360,25 @@ try:
)
except (ValueError, TypeError):
BACKGROUND_HEALTH_CHECK_MAX_TOKENS = None
_background_health_check_max_tokens_reasoning_env = os.getenv(
"BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING"
)
try:
_raw_background_health_check_max_tokens_reasoning = (
_background_health_check_max_tokens_reasoning_env.strip()
if _background_health_check_max_tokens_reasoning_env is not None
else ""
)
BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING: Optional[int] = (
int(_raw_background_health_check_max_tokens_reasoning)
if _raw_background_health_check_max_tokens_reasoning
else None
)
except (ValueError, TypeError):
BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING = None
LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check"
LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli"
LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs"

8
litellm/containers/endpoint_factory.py
View file

@ -90,10 +90,10 @@ def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
custom_llm_provider=resolved_custom_llm_provider,
litellm_params=litellm_params,
)
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
)
if container_provider_config is None:

106
litellm/containers/main.py
View file

@ -168,7 +168,10 @@ def create_container(
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[ContainerObject, Coroutine[Any, Any, ContainerObject],]:
) -> Union[
ContainerObject,
Coroutine[Any, Any, ContainerObject],
]:
"""Create a container using the OpenAI Container API.
Currently supports OpenAI
@ -208,10 +211,10 @@ def create_container(
**kwargs,
)
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if container_provider_config is None:
@ -260,7 +263,7 @@ def create_container(
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
# Encode container_id with provider/model metadata for routing
if isinstance(container_obj, ContainerObject):
container_obj = ContainerRequestUtils.encode_container_id_in_response(
@ -269,7 +272,7 @@ def create_container(
litellm_metadata=kwargs.get("litellm_metadata"),
extra_body=extra_body,
)
return container_obj
except Exception as e:
@ -405,7 +408,10 @@ def list_containers(
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[ContainerListResponse, Coroutine[Any, Any, ContainerListResponse],]:
) -> Union[
ContainerListResponse,
Coroutine[Any, Any, ContainerListResponse],
]:
"""List containers using the OpenAI Container API.
Currently supports OpenAI
@ -434,10 +440,10 @@ def list_containers(
**kwargs,
)
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if container_provider_config is None:
@ -601,7 +607,10 @@ def retrieve_container(
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[ContainerObject, Coroutine[Any, Any, ContainerObject],]:
) -> Union[
ContainerObject,
Coroutine[Any, Any, ContainerObject],
]:
"""Retrieve a container using the OpenAI Container API.
Currently supports OpenAI
@ -630,7 +639,7 @@ def retrieve_container(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
@ -643,10 +652,10 @@ def retrieve_container(
was_encoded = original_container_id != container_id
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
)
if container_provider_config is None:
@ -678,7 +687,7 @@ def retrieve_container(
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
# Encode container_id with provider/model metadata for routing
# If input was encoded, preserve encoding in output using the decoded model_id
if isinstance(container_obj, ContainerObject):
@ -691,14 +700,14 @@ def retrieve_container(
if "model_info" not in litellm_metadata:
litellm_metadata["model_info"] = {}
litellm_metadata["model_info"]["id"] = litellm_params["model_id"]
container_obj = ContainerRequestUtils.encode_container_id_in_response(
response_obj=container_obj,
custom_llm_provider=resolved_custom_llm_provider,
litellm_metadata=litellm_metadata,
extra_body=None,
)
return container_obj
except Exception as e:
@ -822,7 +831,10 @@ def delete_container(
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[DeleteContainerResult, Coroutine[Any, Any, DeleteContainerResult],]:
) -> Union[
DeleteContainerResult,
Coroutine[Any, Any, DeleteContainerResult],
]:
"""Delete a container using the OpenAI Container API.
Currently supports OpenAI
@ -851,7 +863,7 @@ def delete_container(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
@ -864,10 +876,10 @@ def delete_container(
was_encoded = original_container_id != container_id
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
)
if container_provider_config is None:
@ -899,7 +911,7 @@ def delete_container(
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
# Encode container_id in response with provider/model metadata for routing
# If input was encoded, preserve encoding in output using the decoded model_id
if isinstance(delete_result, DeleteContainerResult):
@ -912,14 +924,14 @@ def delete_container(
if "model_info" not in litellm_metadata:
litellm_metadata["model_info"] = {}
litellm_metadata["model_info"]["id"] = litellm_params["model_id"]
delete_result = ContainerRequestUtils.encode_container_id_in_response(
response_obj=delete_result,
custom_llm_provider=resolved_custom_llm_provider,
litellm_metadata=litellm_metadata,
extra_body=None,
)
return delete_result
except Exception as e:
@ -1057,7 +1069,10 @@ def list_container_files(
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[ContainerFileListResponse, Coroutine[Any, Any, ContainerFileListResponse],]:
) -> Union[
ContainerFileListResponse,
Coroutine[Any, Any, ContainerFileListResponse],
]:
"""List files in a container using the OpenAI Container API.
Currently supports OpenAI
@ -1086,7 +1101,7 @@ def list_container_files(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
@ -1095,12 +1110,12 @@ def list_container_files(
litellm_params=litellm_params,
)
)
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
)
if container_provider_config is None:
@ -1285,7 +1300,10 @@ def upload_container_file(
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[ContainerFileObject, Coroutine[Any, Any, ContainerFileObject],]:
) -> Union[
ContainerFileObject,
Coroutine[Any, Any, ContainerFileObject],
]:
"""Upload a file to a container using the OpenAI Container API.
This endpoint allows uploading files directly to a container session,
@ -1343,7 +1361,7 @@ def upload_container_file(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
@ -1352,12 +1370,12 @@ def upload_container_file(
litellm_params=litellm_params,
)
)
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
)
if container_provider_config is None:

9
litellm/containers/utils.py
View file

@ -32,6 +32,7 @@ def decode_managed_container_id_for_request(
return original_container_id, custom_llm_provider, litellm_params
T = TypeVar("T")
@ -129,14 +130,14 @@ class ContainerRequestUtils:
litellm_metadata = litellm_metadata or {}
model_info: Dict[str, Any] = litellm_metadata.get("model_info", {}) or {}
model_id = model_info.get("id")
# Check if we should encode based on routing metadata
should_encode = False
# Case 1: Router/proxy usage (model_id from router)
if model_id is not None:
should_encode = True
# Case 2: target_model_names in extra_body (model-specific routing)
if extra_body and "target_model_names" in extra_body:
should_encode = True
@ -148,7 +149,7 @@ class ContainerRequestUtils:
model_id = target_models.split(",")[0].strip()
elif isinstance(target_models, list) and len(target_models) > 0:
model_id = str(target_models[0]).strip()
# Only encode if we have routing metadata
if should_encode and response_obj and hasattr(response_obj, "id"):
encoded_id = ResponsesAPIRequestUtils._build_container_id(

31
litellm/cost_calculator.py
View file

@ -545,10 +545,9 @@ def cost_per_token( # noqa: PLR0915
model=model, custom_llm_provider=custom_llm_provider
)
if (
(model_info.get("input_cost_per_token") or 0.0) > 0
or (model_info.get("output_cost_per_token") or 0.0) > 0
):
if (model_info.get("input_cost_per_token") or 0.0) > 0 or (
model_info.get("output_cost_per_token") or 0.0
) > 0:
return generic_cost_per_token(
model=model,
usage=usage_block,
@ -1141,9 +1140,9 @@ def completion_cost( # noqa: PLR0915
or isinstance(completion_response, dict)
): # tts returns a custom class
if isinstance(completion_response, dict):
usage_obj: Optional[
Union[dict, Usage]
] = completion_response.get("usage", {})
usage_obj: Optional[Union[dict, Usage]] = (
completion_response.get("usage", {})
)
else:
usage_obj = getattr(completion_response, "usage", {})
if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects(
@ -1606,11 +1605,23 @@ def completion_cost( # noqa: PLR0915
_cache_read_cost: Optional[float] = None
_cache_creation_cost: Optional[float] = None
if cost_per_token_usage_object is not None:
_cr = getattr(cost_per_token_usage_object, "cache_read_input_tokens", None) or (cost_per_token_usage_object.model_extra or {}).get("cache_read_input_tokens")
_cc = getattr(cost_per_token_usage_object, "cache_creation_input_tokens", None) or (cost_per_token_usage_object.model_extra or {}).get("cache_creation_input_tokens")
_cr = getattr(
cost_per_token_usage_object, "cache_read_input_tokens", None
) or (cost_per_token_usage_object.model_extra or {}).get(
"cache_read_input_tokens"
)
_cc = getattr(
cost_per_token_usage_object,
"cache_creation_input_tokens",
None,
) or (cost_per_token_usage_object.model_extra or {}).get(
"cache_creation_input_tokens"
)
if (_cr or _cc) and model:
try:
_mi = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
_mi = litellm.get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
_cr_rate = _mi.get("cache_read_input_token_cost")
if _cr and _cr_rate is not None:
_cache_read_cost = float(_cr) * float(_cr_rate)

88
litellm/evals/main.py
View file

@ -152,10 +152,10 @@ def create_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@ -343,10 +343,10 @@ def list_evals(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@ -513,10 +513,10 @@ def get_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@ -682,10 +682,10 @@ def update_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@ -893,10 +893,10 @@ def delete_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@ -1047,10 +1047,10 @@ def cancel_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@ -1230,10 +1230,10 @@ def create_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@ -1418,10 +1418,10 @@ def list_runs(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@ -1592,10 +1592,10 @@ def get_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@ -1752,10 +1752,10 @@ def cancel_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@ -1921,10 +1921,10 @@ def delete_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:

3
litellm/exceptions.py
View file

@ -281,7 +281,7 @@ class Timeout(openai.APITimeoutError): # type: ignore
return _message
class PermissionDeniedError(openai.PermissionDeniedError): # type:ignore
class PermissionDeniedError(openai.PermissionDeniedError): # type: ignore
def __init__(
self,
message,
@ -847,6 +847,7 @@ class BudgetExceededError(Exception):
):
self.current_cost = current_cost
self.max_budget = max_budget
self.status_code = 429
message = (
message
or f"Budget has been exceeded! Current cost: {current_cost}, Max budget: {max_budget}"

12
litellm/files/main.py
View file

@ -10,7 +10,7 @@ import contextvars
import time
import uuid as uuid_module
from functools import partial
from typing import Any,Coroutine, Dict, Literal, Optional, Union, cast
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
import httpx
@ -53,7 +53,10 @@ from litellm.types.llms.openai import (
OpenAIFileObject,
)
from litellm.types.router import *
from litellm.types.utils import OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS, LlmProviders
from litellm.types.utils import (
OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS,
LlmProviders,
)
from litellm.utils import (
ProviderConfigManager,
client,
@ -73,6 +76,8 @@ def _should_sdk_support_streaming(
Return whether file content streaming is supported for the provider.
"""
return custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS
openai_files_instance = OpenAIFilesAPI()
azure_files_instance = AzureOpenAIFilesAPI()
vertex_ai_files_instance = VertexAIFilesHandler()
@ -1094,9 +1099,10 @@ def file_content_streaming(
)
if asyncio.iscoroutine(response):
async def _await_and_wrap() -> FileContentStreamingResult:
return _wrap_streaming_result(await response)
return _await_and_wrap()
return _wrap_streaming_result(response)
return _wrap_streaming_result(response)

20
litellm/files/streaming.py
View file

@ -1,6 +1,15 @@
import datetime
import traceback
from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, Optional, Union, cast
from typing import (
TYPE_CHECKING,
Any,
AsyncIterator,
Dict,
Iterator,
Optional,
Union,
cast,
)
import anyio
from litellm.files.types import FileContentProvider
@ -11,6 +20,7 @@ if TYPE_CHECKING:
)
from litellm.types.utils import StandardLoggingHiddenParams, StandardLoggingPayload
class FileContentStreamingResponse:
"""
Iterator wrapper for file content streaming that carries LiteLLM metadata
@ -84,7 +94,9 @@ class FileContentStreamingResponse:
self._close_completed = True
self._logging_completed = True
stream_to_close = self.stream_iterator
self.stream_iterator = cast(Union[Iterator[bytes], AsyncIterator[bytes]], iter(()))
self.stream_iterator = cast(
Union[Iterator[bytes], AsyncIterator[bytes]], iter(())
)
# Shield cleanup from request cancellation so upstream HTTP connections
# are released promptly on client disconnects.
@ -103,7 +115,9 @@ class FileContentStreamingResponse:
self._close_completed = True
self._logging_completed = True
stream_to_close = self.stream_iterator
self.stream_iterator = cast(Union[Iterator[bytes], AsyncIterator[bytes]], iter(()))
self.stream_iterator = cast(
Union[Iterator[bytes], AsyncIterator[bytes]], iter(())
)
if hasattr(stream_to_close, "close"):
cast(Iterator[bytes], stream_to_close).close() # type: ignore[attr-defined]

39
litellm/images/main.py
View file

@ -210,7 +210,10 @@ def image_generation( # noqa: PLR0915
api_version: Optional[str] = None,
custom_llm_provider=None,
**kwargs,
) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]:
) -> Union[
ImageResponse,
Coroutine[Any, Any, ImageResponse],
]:
"""
Maps the https://api.openai.com/v1/images/generations endpoint.
@ -864,11 +867,11 @@ def image_edit( # noqa: PLR0915
)
# get provider config
image_edit_provider_config: Optional[
BaseImageEditConfig
] = ProviderConfigManager.get_provider_image_edit_config(
model=model,
provider=litellm.LlmProviders(custom_llm_provider),
image_edit_provider_config: Optional[BaseImageEditConfig] = (
ProviderConfigManager.get_provider_image_edit_config(
model=model,
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if image_edit_provider_config is None:
@ -876,20 +879,20 @@ def image_edit( # noqa: PLR0915
local_vars.update(kwargs)
# Get ImageEditOptionalRequestParams with only valid parameters
image_edit_optional_params: ImageEditOptionalRequestParams = (
_get_ImageEditRequestUtils().get_requested_image_edit_optional_param(
local_vars
)
image_edit_optional_params: (
ImageEditOptionalRequestParams
) = _get_ImageEditRequestUtils().get_requested_image_edit_optional_param(
local_vars
)
# Get optional parameters for the responses API
image_edit_request_params: Dict = (
_get_ImageEditRequestUtils().get_optional_params_image_edit(
model=model,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_params=image_edit_optional_params,
drop_params=kwargs.get("drop_params"),
additional_drop_params=kwargs.get("additional_drop_params"),
)
image_edit_request_params: (
Dict
) = _get_ImageEditRequestUtils().get_optional_params_image_edit(
model=model,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_params=image_edit_optional_params,
drop_params=kwargs.get("drop_params"),
additional_drop_params=kwargs.get("additional_drop_params"),
)
# Pre Call logging

8
litellm/integrations/SlackAlerting/hanging_request_check.py
View file

@ -102,10 +102,10 @@ class AlertingHangingRequestCheck:
)
for request_id in hanging_requests:
hanging_request_data: Optional[
HangingRequestData
] = await self.hanging_request_cache.async_get_cache(
key=request_id,
hanging_request_data: Optional[HangingRequestData] = (
await self.hanging_request_cache.async_get_cache(
key=request_id,
)
)
if hanging_request_data is None:

18
litellm/integrations/SlackAlerting/slack_alerting.py
View file

@ -852,9 +852,9 @@ class SlackAlerting(CustomBatchLogger):
### UNIQUE CACHE KEY ###
cache_key = provider + region_name
outage_value: Optional[
ProviderRegionOutageModel
] = await self.internal_usage_cache.async_get_cache(key=cache_key)
outage_value: Optional[ProviderRegionOutageModel] = (
await self.internal_usage_cache.async_get_cache(key=cache_key)
)
# Convert deployment_ids back to set if it was stored as a list
if outage_value is not None:
@ -1443,9 +1443,9 @@ Model Info:
self.alert_to_webhook_url is not None
and alert_type in self.alert_to_webhook_url
):
_digest_webhook: Optional[
Union[str, List[str]]
] = self.alert_to_webhook_url[alert_type]
_digest_webhook: Optional[Union[str, List[str]]] = (
self.alert_to_webhook_url[alert_type]
)
elif self.default_webhook_url is not None:
_digest_webhook = self.default_webhook_url
else:
@ -1499,9 +1499,9 @@ Model Info:
self.alert_to_webhook_url is not None
and alert_type in self.alert_to_webhook_url
):
slack_webhook_url: Optional[
Union[str, List[str]]
] = self.alert_to_webhook_url[alert_type]
slack_webhook_url: Optional[Union[str, List[str]]] = (
self.alert_to_webhook_url[alert_type]
)
elif self.default_webhook_url is not None:
slack_webhook_url = self.default_webhook_url
else:

1
litellm/integrations/agentops/agentops.py
View file

@ -1,6 +1,7 @@
"""
AgentOps integration for LiteLLM - Provides OpenTelemetry tracing for LLM calls
"""
import os
from dataclasses import dataclass
from typing import Optional, Dict, Any

8
litellm/integrations/anthropic_cache_control_hook.py
View file

@ -106,10 +106,10 @@ class AnthropicCacheControlHook(CustomPromptManagement):
targetted_index += len(messages)
if 0 <= targetted_index < len(messages):
messages[
targetted_index
] = AnthropicCacheControlHook._safe_insert_cache_control_in_message(
messages[targetted_index], control
messages[targetted_index] = (
AnthropicCacheControlHook._safe_insert_cache_control_in_message(
messages[targetted_index], control
)
)
else:
verbose_logger.warning(

6
litellm/integrations/arize/arize_phoenix.py
View file

@ -178,9 +178,9 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
start_time_val = kwargs.get("start_time", kwargs.get("api_call_start_time"))
parent_span = self.tracer.start_span(
name="litellm_proxy_request",
start_time=self._to_ns(start_time_val)
if start_time_val is not None
else None,
start_time=(
self._to_ns(start_time_val) if start_time_val is not None else None
),
context=traceparent_ctx,
kind=self.span_kind.SERVER,
)

12
litellm/integrations/azure_storage/azure_storage.py
View file

@ -54,12 +54,12 @@ class AzureBlobStorageLogger(CustomBatchLogger):
self._service_client_timeout: Optional[float] = None
# Internal variables used for Token based authentication
self.azure_auth_token: Optional[
str
] = None # the Azure AD token to use for Azure Storage API requests
self.token_expiry: Optional[
datetime
] = None # the expiry time of the currentAzure AD token
self.azure_auth_token: Optional[str] = (
None # the Azure AD token to use for Azure Storage API requests
)
self.token_expiry: Optional[datetime] = (
None # the expiry time of the currentAzure AD token
)
asyncio.create_task(self.periodic_flush())
self.flush_lock = asyncio.Lock()

6
litellm/integrations/braintrust_logging.py
View file

@ -52,9 +52,9 @@ class BraintrustLogger(CustomLogger):
"Authorization": "Bearer " + self.api_key,
"Content-Type": "application/json",
}
self._project_id_cache: Dict[
str, str
] = {} # Cache mapping project names to IDs
self._project_id_cache: Dict[str, str] = (
{}
) # Cache mapping project names to IDs
self.global_braintrust_http_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)

8
litellm/integrations/cloudzero/cloudzero.py
View file

@ -402,10 +402,10 @@ class CloudZeroLogger(CustomLogger):
from litellm.constants import CLOUDZERO_EXPORT_INTERVAL_MINUTES
from litellm.integrations.custom_logger import CustomLogger
prometheus_loggers: List[
CustomLogger
] = litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=CloudZeroLogger
prometheus_loggers: List[CustomLogger] = (
litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=CloudZeroLogger
)
)
# we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them
verbose_logger.debug("found %s cloudzero loggers", len(prometheus_loggers))

12
litellm/integrations/cloudzero/transform.py
View file

@ -159,9 +159,9 @@ class CBFTransformer:
# CloudZero CBF format with proper column names
cbf_record = {
# Required CBF fields
"time/usage_start": usage_date.isoformat()
if usage_date
else None, # Required: ISO-formatted UTC datetime
"time/usage_start": (
usage_date.isoformat() if usage_date else None
), # Required: ISO-formatted UTC datetime
"cost/cost": float(row.get("spend", 0.0)), # Required: billed cost
"resource/id": resource_id, # CZRN (CloudZero Resource Name)
# Usage metrics for token consumption
@ -182,9 +182,9 @@ class CBFTransformer:
# Add CZRN components that don't have direct CBF column mappings as resource tags
cbf_record["resource/tag:provider"] = provider # CZRN provider component
cbf_record[
"resource/tag:model"
] = cloud_local_id # CZRN cloud-local-id component (model)
cbf_record["resource/tag:model"] = (
cloud_local_id # CZRN cloud-local-id component (model)
)
# Add resource tags for all dimensions (using resource/tag:<key> format)
for key, value in dimensions.items():

14
litellm/integrations/compression_interception/__init__.py Normal file
View file

@ -0,0 +1,14 @@
"""
Compression Interception Module
Provides server-side prompt compression + retrieval tool fulfillment for
Anthropic Messages agentic loops.
"""
from litellm.integrations.compression_interception.handler import (
CompressionInterceptionLogger,
)
__all__ = [
"CompressionInterceptionLogger",
]

399
litellm/integrations/compression_interception/handler.py Normal file
View file

@ -0,0 +1,399 @@
"""
Compression Interception Handler
CustomLogger that compresses inbound Anthropic Messages requests and fulfills
litellm_content_retrieve tool calls server-side via the typed agentic loop plan.
"""
import time
import uuid
from typing import Any, Dict, List, Optional, Tuple, cast
from litellm._logging import verbose_logger
from litellm.compression import compress
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.integrations.compression_interception import (
CompressionInterceptionConfig,
)
from litellm.types.integrations.custom_logger import (
AgenticLoopPlan,
AgenticLoopRequestPatch,
)
from litellm.types.utils import CallTypes
LITELLM_CONTENT_RETRIEVE_TOOL_NAME = "litellm_content_retrieve"
_CACHE_TTL_SECONDS = 15 * 60
class CompressionInterceptionLogger(CustomLogger):
"""
CustomLogger that implements transparent prompt compression + retrieval loops.
Flow:
1. Compress inbound /v1/messages requests in pre-call hook.
2. Inject litellm_content_retrieve tool and persist compressed cache by call_id.
3. Detect retrieval tool_use blocks in first model response.
4. Build typed rerun plan with tool_result blocks from the compressed cache.
"""
def __init__(
self,
enabled: bool = True,
compression_trigger: int = 200_000,
compression_target: Optional[int] = None,
embedding_model: Optional[str] = None,
embedding_model_params: Optional[Dict[str, Any]] = None,
):
super().__init__()
self.enabled = enabled
self.compression_trigger = compression_trigger
self.compression_target = compression_target
self.embedding_model = embedding_model
self.embedding_model_params = embedding_model_params
self._compression_cache_by_call_id: Dict[str, Tuple[Dict[str, str], float]] = {}
@classmethod
def from_config_yaml(
cls, config: CompressionInterceptionConfig
) -> "CompressionInterceptionLogger":
return cls(
enabled=bool(config.get("enabled", True)),
compression_trigger=int(config.get("compression_trigger", 200_000)),
compression_target=config.get("compression_target"),
embedding_model=config.get("embedding_model"),
embedding_model_params=config.get("embedding_model_params"),
)
@staticmethod
def initialize_from_proxy_config(
litellm_settings: Dict[str, Any],
callback_specific_params: Dict[str, Any],
) -> "CompressionInterceptionLogger":
compression_params: CompressionInterceptionConfig = {}
if "compression_interception_params" in litellm_settings:
compression_params = litellm_settings["compression_interception_params"]
elif "compression_interception" in callback_specific_params:
compression_params = callback_specific_params["compression_interception"]
return CompressionInterceptionLogger.from_config_yaml(compression_params)
async def async_pre_call_deployment_hook(
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
) -> Optional[dict]:
if not self.enabled:
return None
if call_type is not None and call_type != CallTypes.anthropic_messages:
return None
if int(kwargs.get("_agentic_loop_depth", 0) or 0) > 0:
return None
messages = kwargs.get("messages")
model = kwargs.get("model")
if not isinstance(messages, list) or not isinstance(model, str):
return None
if self._has_retrieval_tool(kwargs.get("tools")):
return None
self._prune_expired_cache()
compressed = compress( # type: ignore
messages=messages,
model=model,
call_type=CallTypes.anthropic_messages,
compression_trigger=self.compression_trigger,
compression_target=self.compression_target,
embedding_model=self.embedding_model,
embedding_model_params=self.embedding_model_params,
)
cache = cast(Dict[str, str], compressed.get("cache", {}))
skip_reason = cast(Optional[str], compressed.get("compression_skipped_reason"))
compressed_tools = cast(List[Dict[str, Any]], compressed.get("tools", []))
# Only mutate kwargs when compression actually produced a result.
# If compression was a no-op (below trigger, invalid tool sequence, etc.),
# leave ``messages`` and ``tools`` untouched — injecting an empty
# ``tools: []`` onto a request that originally had no tools breaks
# Anthropic Messages requests.
if cache:
kwargs["messages"] = compressed["messages"]
if compressed_tools:
kwargs["tools"] = self._merge_tools(
existing_tools=cast(
Optional[List[Dict[str, Any]]], kwargs.get("tools")
),
compressed_tools=compressed_tools,
)
call_id = cast(Optional[str], kwargs.get("litellm_call_id"))
if not call_id:
call_id = str(uuid.uuid4())
kwargs["litellm_call_id"] = call_id
self._compression_cache_by_call_id[call_id] = (cache, time.time())
verbose_logger.debug(
"CompressionInterception: compressed request [call_id=%s original=%d compressed=%d cached_keys=%d]",
call_id,
compressed.get("original_tokens"),
compressed.get("compressed_tokens"),
len(cache),
)
elif skip_reason is not None:
verbose_logger.debug(
"CompressionInterception: compression skipped [reason=%s original=%d compressed=%d]",
skip_reason,
compressed.get("original_tokens"),
compressed.get("compressed_tokens"),
)
return kwargs
async def async_should_run_agentic_loop(
self,
response: Any,
model: str,
messages: List[Dict],
tools: Optional[List[Dict]],
stream: bool,
custom_llm_provider: str,
kwargs: Dict,
) -> Tuple[bool, Dict]:
if not self.enabled:
return False, {}
if not self._has_retrieval_tool(tools):
return False, {}
tool_calls, thinking_blocks = self._extract_retrieval_tool_calls(
response=response
)
if not tool_calls:
return False, {}
return True, {
"tool_calls": tool_calls,
"thinking_blocks": thinking_blocks,
"tool_type": "compression_retrieval",
}
async def async_build_agentic_loop_plan(
self,
tools: Dict,
model: str,
messages: List[Dict],
response: Any,
anthropic_messages_provider_config: Any,
anthropic_messages_optional_request_params: Dict,
logging_obj: Any,
stream: bool,
kwargs: Dict,
) -> AgenticLoopPlan:
self._prune_expired_cache()
tool_calls = cast(List[Dict[str, Any]], tools.get("tool_calls", []))
thinking_blocks = cast(List[Dict[str, Any]], tools.get("thinking_blocks", []))
call_id = self._resolve_call_id(logging_obj=logging_obj, kwargs=kwargs)
cache = self._get_cache(call_id=call_id)
retrieval_results = [
self._resolve_retrieval_content(tc, cache) for tc in tool_calls
]
assistant_message = {
"role": "assistant",
"content": thinking_blocks
+ [
{
"type": "tool_use",
"id": tc.get("id"),
"name": tc.get("name", LITELLM_CONTENT_RETRIEVE_TOOL_NAME),
"input": tc.get("input", {}),
}
for tc in tool_calls
],
}
user_message = {
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": tool_calls[i].get("id"),
"content": retrieval_results[i],
}
for i in range(len(tool_calls))
],
}
follow_up_messages = messages + [assistant_message, user_message]
max_tokens = cast(
Optional[int],
anthropic_messages_optional_request_params.get("max_tokens")
or kwargs.get("max_tokens"),
)
optional_params_without_max_tokens = {
k: v
for k, v in anthropic_messages_optional_request_params.items()
if k != "max_tokens"
}
full_model_name = model
if logging_obj is not None:
agentic_params = logging_obj.model_call_details.get(
"agentic_loop_params", {}
)
full_model_name = cast(str, agentic_params.get("model", model))
request_patch = AgenticLoopRequestPatch(
model=full_model_name,
messages=follow_up_messages,
max_tokens=max_tokens,
optional_params=optional_params_without_max_tokens,
kwargs=self._prepare_followup_kwargs(kwargs=kwargs),
)
return AgenticLoopPlan(
run_agentic_loop=True,
request_patch=request_patch,
metadata={"tool_type": "compression_retrieval", "call_id": call_id or ""},
)
def _prune_expired_cache(self) -> None:
now = time.time()
self._compression_cache_by_call_id = {
call_id: (cache, created_at)
for call_id, (
cache,
created_at,
) in self._compression_cache_by_call_id.items()
if now - created_at <= _CACHE_TTL_SECONDS
}
def _get_cache(self, call_id: Optional[str]) -> Dict[str, str]:
if not call_id:
return {}
cache_entry = self._compression_cache_by_call_id.get(call_id)
if cache_entry is None:
return {}
return cache_entry[0]
def _resolve_call_id(
self, logging_obj: Any, kwargs: Dict[str, Any]
) -> Optional[str]:
if logging_obj is not None:
logging_call_id = getattr(logging_obj, "litellm_call_id", None)
if isinstance(logging_call_id, str) and logging_call_id:
return logging_call_id
kwargs_call_id = kwargs.get("litellm_call_id")
return cast(
Optional[str], kwargs_call_id if isinstance(kwargs_call_id, str) else None
)
def _resolve_retrieval_content(
self, tool_call: Dict[str, Any], cache: Dict[str, str]
) -> str:
raw_input = tool_call.get("input", {})
key = ""
if isinstance(raw_input, dict):
key = str(raw_input.get("key", "") or "")
if not key:
return "No retrieval key provided."
if key in cache:
return cache[key]
return f"[compressed content key '{key}' not found]"
def _extract_retrieval_tool_calls(
self, response: Any
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
if isinstance(response, dict):
content = response.get("content", [])
else:
content = getattr(response, "content", []) or []
if not isinstance(content, list):
return [], []
tool_calls: List[Dict[str, Any]] = []
thinking_blocks: List[Dict[str, Any]] = []
for block in content:
if isinstance(block, dict):
block_type = block.get("type")
block_name = block.get("name")
if block_type in ("thinking", "redacted_thinking"):
thinking_blocks.append(block)
if (
block_type == "tool_use"
and block_name == LITELLM_CONTENT_RETRIEVE_TOOL_NAME
):
tool_calls.append(
{
"id": block.get("id"),
"type": "tool_use",
"name": block_name,
"input": block.get("input", {}),
}
)
else:
block_type = getattr(block, "type", None)
block_name = getattr(block, "name", None)
if block_type == "thinking":
thinking_blocks.append(
{
"type": "thinking",
"thinking": getattr(block, "thinking", ""),
"signature": getattr(block, "signature", ""),
}
)
elif block_type == "redacted_thinking":
thinking_blocks.append(
{
"type": "redacted_thinking",
"data": getattr(block, "data", ""),
}
)
if (
block_type == "tool_use"
and block_name == LITELLM_CONTENT_RETRIEVE_TOOL_NAME
):
tool_calls.append(
{
"id": getattr(block, "id", None),
"type": "tool_use",
"name": block_name,
"input": getattr(block, "input", {}) or {},
}
)
return tool_calls, thinking_blocks
def _prepare_followup_kwargs(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
internal_keys = {"litellm_logging_obj"}
return {
k: v
for k, v in kwargs.items()
if not k.startswith("_compression_interception") and k not in internal_keys
}
def _has_retrieval_tool(self, tools: Any) -> bool:
if not isinstance(tools, list):
return False
for tool in tools:
if not isinstance(tool, dict):
continue
function = tool.get("function")
if tool.get("type") == "function" and isinstance(function, dict):
if function.get("name") == LITELLM_CONTENT_RETRIEVE_TOOL_NAME:
return True
if (
tool.get("type") == "custom"
and tool.get("name") == LITELLM_CONTENT_RETRIEVE_TOOL_NAME
):
return True
return False
def _merge_tools(
self,
existing_tools: Optional[List[Dict[str, Any]]],
compressed_tools: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
merged = list(existing_tools or [])
if self._has_retrieval_tool(merged):
return merged
merged.extend(compressed_tools)
return merged

51
litellm/integrations/custom_guardrail.py
View file

@ -255,26 +255,44 @@ class CustomGuardrail(CustomLogger):
f"Event hook {event_hook} is not in the supported event hooks {supported_event_hooks}"
)
@staticmethod
def _get_admin_metadata(data: dict) -> dict:
"""Return merged admin-configured key and team metadata from the request data.
The proxy may inject admin metadata (user_api_key_metadata,
user_api_key_team_metadata) into either ``metadata`` or
``litellm_metadata`` depending on endpoint. Check both so a caller
cannot shadow admin config by pre-populating the other key.
Key-level settings override team-level.
"""
team_meta: dict = {}
key_meta: dict = {}
for key in ("metadata", "litellm_metadata"):
# Defensive: an unparsed JSON-string metadata could leak past the
# proxy's normal parse path; don't AttributeError on .get().
meta = data.get(key)
if not isinstance(meta, dict):
continue
team_meta = meta.get("user_api_key_team_metadata") or team_meta
key_meta = meta.get("user_api_key_metadata") or key_meta
return {**team_meta, **key_meta}
def get_disable_global_guardrail(self, data: dict) -> Optional[bool]:
"""
Returns True if the global guardrail should be disabled
Returns True if the global guardrail should be disabled.
Reads from admin-configured key/team metadata only, not from
the request body, to prevent callers from disabling guardrails.
"""
if "disable_global_guardrails" in data:
return data["disable_global_guardrails"]
metadata = data.get("litellm_metadata") or data.get("metadata", {})
if "disable_global_guardrails" in metadata:
return metadata["disable_global_guardrails"]
return False
return self._get_admin_metadata(data).get("disable_global_guardrails", False)
def get_opted_out_global_guardrails_from_metadata(self, data: dict) -> List[str]:
"""
Returns the list of global guardrail names the team/key has opted out of.
Reads from admin-configured key/team metadata only.
"""
if "opted_out_global_guardrails" in data:
value = data["opted_out_global_guardrails"]
return value if isinstance(value, list) else []
metadata = data.get("litellm_metadata") or data.get("metadata", {})
value = metadata.get("opted_out_global_guardrails")
value = self._get_admin_metadata(data).get("opted_out_global_guardrails")
return value if isinstance(value, list) else []
def _is_valid_response_type(self, result: Any) -> bool:
@ -417,7 +435,9 @@ class CustomGuardrail(CustomLogger):
"""
requested_guardrails = self.get_guardrail_from_metadata(data)
disable_global_guardrail = self.get_disable_global_guardrail(data)
opted_out_global_guardrails = self.get_opted_out_global_guardrails_from_metadata(data)
opted_out_global_guardrails = (
self.get_opted_out_global_guardrails_from_metadata(data)
)
verbose_logger.debug(
"inside should_run_guardrail for guardrail=%s event_type= %s guardrail_supported_event_hooks= %s requested_guardrails= %s self.default_on= %s",
self.guardrail_name,
@ -426,7 +446,10 @@ class CustomGuardrail(CustomLogger):
requested_guardrails,
self.default_on,
)
if self.default_on is True and self.guardrail_name in opted_out_global_guardrails:
if (
self.default_on is True
and self.guardrail_name in opted_out_global_guardrails
):
return False
if self.default_on is True and disable_global_guardrail is not True:

45
litellm/integrations/custom_logger.py
View file

@ -20,6 +20,7 @@ from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
from litellm.types.integrations.argilla import ArgillaItem
from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.integrations.custom_logger import AgenticLoopPlan
from litellm.types.utils import (
AdapterCompletionStreamWrapper,
CallTypes,
@ -239,7 +240,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
self,
model: str,
request_kwargs: Dict,
messages: Optional[List[Dict[str, str]]] = None,
messages: Optional[List[Dict[str, Any]]] = None,
input: Optional[Union[str, List]] = None,
specific_deployment: Optional[bool] = False,
) -> Optional[PreRoutingHookResponse]:
@ -676,6 +677,26 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
"""
pass
async def async_build_agentic_loop_plan(
self,
tools: Dict,
model: str,
messages: List[Dict],
response: Any,
anthropic_messages_provider_config: Any,
anthropic_messages_optional_request_params: Dict,
logging_obj: "LiteLLMLoggingObj",
stream: bool,
kwargs: Dict,
) -> AgenticLoopPlan:
"""
Build a typed rerun plan for Anthropic Messages agentic loops.
Override this method to separate callback decision/tool execution from
follow-up request execution (handled by BaseLLMHTTPHandler).
"""
return AgenticLoopPlan(run_agentic_loop=False)
async def async_should_run_chat_completion_agentic_loop(
self,
response: Any,
@ -707,6 +728,22 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
"""
pass
async def async_build_chat_completion_agentic_loop_plan(
self,
tools: Dict,
model: str,
messages: List[Dict],
response: Any,
optional_params: Dict,
logging_obj: "LiteLLMLoggingObj",
stream: bool,
kwargs: Dict,
) -> AgenticLoopPlan:
"""
Build a typed rerun plan for chat-completions agentic loops.
"""
return AgenticLoopPlan(run_agentic_loop=False)
# Useful helpers for custom logger classes
def truncate_standard_logging_payload_content(
@ -874,9 +911,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
model_response_dict = model_response.model_dump()
standard_logging_object_copy["response"] = model_response_dict
model_call_details_copy[
"standard_logging_object"
] = standard_logging_object_copy
model_call_details_copy["standard_logging_object"] = (
standard_logging_object_copy
)
return model_call_details_copy
async def get_proxy_server_request_from_cold_storage_with_object_key(

24
litellm/integrations/datadog/datadog_llm_obs.py
View file

@ -349,9 +349,9 @@ class DataDogLLMObsLogger(CustomBatchLogger):
if standard_logging_payload.get("status") == "failure":
# Try to get structured error information first
error_information: Optional[
StandardLoggingPayloadErrorInformation
] = standard_logging_payload.get("error_information")
error_information: Optional[StandardLoggingPayloadErrorInformation] = (
standard_logging_payload.get("error_information")
)
if error_information:
error_info = DDLLMObsError(
@ -621,9 +621,9 @@ class DataDogLLMObsLogger(CustomBatchLogger):
latency_metrics["litellm_overhead_time_ms"] = litellm_overhead_ms
# Guardrail overhead latency
guardrail_info: Optional[
list[StandardLoggingGuardrailInformation]
] = standard_logging_payload.get("guardrail_information")
guardrail_info: Optional[list[StandardLoggingGuardrailInformation]] = (
standard_logging_payload.get("guardrail_information")
)
if guardrail_info is not None:
total_duration = 0.0
for info in guardrail_info:
@ -793,15 +793,15 @@ class DataDogLLMObsLogger(CustomBatchLogger):
if function_arguments:
# Store arguments as JSON string for Datadog
if isinstance(function_arguments, str):
kv_pairs[
f"tool_calls.{idx}.function.arguments"
] = function_arguments
kv_pairs[f"tool_calls.{idx}.function.arguments"] = (
function_arguments
)
else:
import json
kv_pairs[
f"tool_calls.{idx}.function.arguments"
] = json.dumps(function_arguments)
kv_pairs[f"tool_calls.{idx}.function.arguments"] = (
json.dumps(function_arguments)
)
except (KeyError, TypeError, ValueError) as e:
verbose_logger.debug(
f"DataDogLLMObs: Error processing tool call {idx}: {str(e)}"

6
litellm/integrations/gcs_bucket/gcs_bucket_base.py
View file

@ -150,9 +150,9 @@ class GCSBucketBase(CustomBatchLogger):
if kwargs is None:
kwargs = {}
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params", None)
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
bucket_name: str
path_service_account: Optional[str]

6
litellm/integrations/humanloop.py
View file

@ -162,7 +162,11 @@ class HumanloopLogger(CustomLogger):
prompt_version: Optional[int] = None,
ignore_prompt_manager_model: Optional[bool] = False,
ignore_prompt_manager_optional_params: Optional[bool] = False,
) -> Tuple[str, List[AllMessageValues], dict,]:
) -> Tuple[
str,
List[AllMessageValues],
dict,
]:
humanloop_api_key = dynamic_callback_params.get(
"humanloop_api_key"
) or get_secret_str("HUMANLOOP_API_KEY")

6
litellm/integrations/langfuse/langfuse.py
View file

@ -572,9 +572,9 @@ class LangFuseLogger:
# we clean out all extra litellm metadata params before logging
clean_metadata: Dict[str, Any] = {}
if prompt_management_metadata is not None:
clean_metadata[
"prompt_management_metadata"
] = prompt_management_metadata
clean_metadata["prompt_management_metadata"] = (
prompt_management_metadata
)
if isinstance(metadata, dict):
for key, value in metadata.items():
# generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy

4
litellm/integrations/langfuse/langfuse_handler.py
View file

@ -86,9 +86,7 @@ class LangFuseHandler:
if globalLangfuseLogger is not None:
return globalLangfuseLogger
credentials_dict: Dict[
str, Any
] = (
credentials_dict: Dict[str, Any] = (
{}
) # the global langfuse logger uses Environment Variables, there are no dynamic credentials
globalLangfuseLogger = in_memory_dynamic_logger_cache.get_cache(

6
litellm/integrations/langfuse/langfuse_prompt_management.py
View file

@ -190,7 +190,11 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
prompt_version: Optional[int] = None,
ignore_prompt_manager_model: Optional[bool] = False,
ignore_prompt_manager_optional_params: Optional[bool] = False,
) -> Tuple[str, List[AllMessageValues], dict,]:
) -> Tuple[
str,
List[AllMessageValues],
dict,
]:
return self.get_chat_completion_prompt(
model,
messages,

18
litellm/integrations/langsmith.py
View file

@ -83,9 +83,9 @@ class LangsmithLogger(CustomBatchLogger):
if _batch_size:
self.batch_size = int(_batch_size)
self.log_queue: List[LangsmithQueueObject] = []
self._flush_task: Optional[
asyncio.Task[Any]
] = self._start_periodic_flush_task()
self._flush_task: Optional[asyncio.Task[Any]] = (
self._start_periodic_flush_task()
)
def _start_periodic_flush_task(self) -> Optional[asyncio.Task[Any]]:
"""Start the periodic flush task only when an event loop is already running."""
@ -501,9 +501,9 @@ class LangsmithLogger(CustomBatchLogger):
return log_queue_by_credentials
def _get_sampling_rate_to_use_for_request(self, kwargs: Dict[str, Any]) -> float:
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params", None)
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
sampling_rate: float = self.sampling_rate
if standard_callback_dynamic_params is not None:
_sampling_rate = standard_callback_dynamic_params.get(
@ -523,9 +523,9 @@ class LangsmithLogger(CustomBatchLogger):
Otherwise, use the default credentials.
"""
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params", None)
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
if standard_callback_dynamic_params is not None:
credentials = self.get_credentials_from_env(
langsmith_api_key=standard_callback_dynamic_params.get(

6
litellm/integrations/mock_client_factory.py
View file

@ -25,9 +25,9 @@ class MockClientConfig:
default_latency_ms: int = 100 # Default mock latency in milliseconds
default_status_code: int = 200 # Default HTTP status code
default_json_data: Optional[Dict] = None # Default JSON response data
url_matchers: Optional[
List[str]
] = None # List of strings to match in URLs (e.g., ["storage.googleapis.com"])
url_matchers: Optional[List[str]] = (
None # List of strings to match in URLs (e.g., ["storage.googleapis.com"])
)
patch_async_handler: bool = True # Whether to patch AsyncHTTPHandler.post
patch_sync_client: bool = False # Whether to patch httpx.Client.post
patch_http_handler: bool = (

6
litellm/integrations/opentelemetry.py
View file

@ -655,9 +655,9 @@ class OpenTelemetry(CustomLogger):
def _get_dynamic_otel_headers_from_kwargs(self, kwargs) -> Optional[dict]:
"""Extract dynamic headers from kwargs if available."""
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params")
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params")
)
if not standard_callback_dynamic_params:
return None

6
litellm/integrations/posthog.py
View file

@ -349,9 +349,9 @@ class PostHogLogger(CustomBatchLogger):
Returns:
tuple[str, str]: (api_key, api_url)
"""
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params", None)
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
if standard_callback_dynamic_params is not None:
api_key = (

402
litellm/integrations/prometheus.py
View file

@ -1,6 +1,8 @@
# used for /metrics endpoint on LiteLLM Proxy
#### What this does ####
# On success, log events to Prometheus
from __future__ import annotations
import asyncio
import os
import sys
@ -14,6 +16,7 @@ from typing import (
List,
Literal,
Optional,
Sequence,
Tuple,
Union,
cast,
@ -22,6 +25,10 @@ from typing import (
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.prometheus_helpers import (
PrometheusLabelFactoryContext,
_get_cached_end_user_id_for_cost_tracking,
)
from litellm.litellm_core_utils.core_helpers import (
get_litellm_metadata_from_kwargs,
get_metadata_variable_name_from_kwargs,
@ -44,24 +51,6 @@ if TYPE_CHECKING:
else:
AsyncIOScheduler = Any
# Cached lazy import for get_end_user_id_for_cost_tracking
# Module-level cache to avoid repeated imports while preserving memory benefits
_get_end_user_id_for_cost_tracking = None
def _get_cached_end_user_id_for_cost_tracking():
"""
Get cached get_end_user_id_for_cost_tracking function.
Lazy imports on first call to avoid loading utils.py at import time (60MB saved).
Subsequent calls use cached function for better performance.
"""
global _get_end_user_id_for_cost_tracking
if _get_end_user_id_for_cost_tracking is None:
from litellm.utils import get_end_user_id_for_cost_tracking
_get_end_user_id_for_cost_tracking = get_end_user_id_for_cost_tracking
return _get_end_user_id_for_cost_tracking
class PrometheusLogger(CustomLogger):
# Class variables or attributes
@ -88,7 +77,9 @@ class PrometheusLogger(CustomLogger):
_custom_buckets = litellm.prometheus_latency_buckets
self.latency_buckets = (
tuple(_custom_buckets) if _custom_buckets is not None else LATENCY_BUCKETS
tuple(_custom_buckets)
if _custom_buckets is not None
else LATENCY_BUCKETS
)
# Create metric factory functions
@ -573,7 +564,6 @@ class PrometheusLogger(CustomLogger):
self.enabled_metrics = set()
for group_config in config:
# Validate configuration using Pydantic
if isinstance(group_config, dict):
parsed_config = PrometheusMetricsConfig(**group_config)
else:
@ -993,12 +983,26 @@ class PrometheusLogger(CustomLogger):
return filtered_labels
def _inc_labeled_counter(
self,
counter: Any,
metric_name: DEFINED_PROMETHEUS_METRICS,
enum_values: UserAPIKeyLabelValues,
label_context: Optional[PrometheusLabelFactoryContext] = None,
amount: float = 1.0,
) -> None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name=metric_name),
enum_values=enum_values,
label_context=label_context,
)
counter.labels(**_labels).inc(amount)
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
# Define prometheus client
from litellm.types.utils import StandardLoggingPayload
verbose_logger.debug(
f"prometheus Logging - Enters success logging function for kwargs {kwargs}"
"prometheus Logging - Enters success logging function (kwargs keys: %s)",
list(kwargs.keys()) if isinstance(kwargs, dict) else type(kwargs).__name__,
)
# unpack kwargs
@ -1097,9 +1101,11 @@ class PrometheusLogger(CustomLogger):
),
client_ip=standard_logging_payload["metadata"].get("requester_ip_address"),
user_agent=standard_logging_payload["metadata"].get("user_agent"),
stream=str(standard_logging_payload.get("stream"))
if litellm.prometheus_emit_stream_label
else None,
stream=(
str(standard_logging_payload.get("stream"))
if litellm.prometheus_emit_stream_label
else None
),
)
if (
@ -1111,6 +1117,10 @@ class PrometheusLogger(CustomLogger):
user_api_key = hash_token(user_api_key)
label_context = PrometheusLabelFactoryContext(
enum_values
) # amortized per request.
# increment total LLM requests and spend metric
self._increment_top_level_request_and_spend_metrics(
end_user_id=end_user_id,
@ -1122,6 +1132,7 @@ class PrometheusLogger(CustomLogger):
user_id=user_id,
response_cost=response_cost,
enum_values=enum_values,
label_context=label_context,
)
# input, output, total token metrics
@ -1138,6 +1149,7 @@ class PrometheusLogger(CustomLogger):
user_api_team_alias=user_api_team_alias,
user_id=user_id,
enum_values=enum_values,
label_context=label_context,
)
# remaining budget metrics
@ -1173,29 +1185,36 @@ class PrometheusLogger(CustomLogger):
# 1. We just checked if isinstance(standard_logging_payload, dict). Pyright complains.
# 2. Pyright does not allow us to run isinstance(standard_logging_payload, StandardLoggingPayload) <- this would be ideal
enum_values=enum_values,
label_context=label_context,
)
# set x-ratelimit headers
self.set_llm_deployment_success_metrics(
kwargs, start_time, end_time, enum_values, output_tokens
kwargs,
start_time,
end_time,
enum_values,
output_tokens,
label_context=label_context,
)
# cache metrics
self._increment_cache_metrics(
standard_logging_payload=standard_logging_payload, # type: ignore
enum_values=enum_values,
label_context=label_context,
)
# increment litellm_proxy_total_requests_metric for all successful requests
# (both streaming and non-streaming) in this single location to prevent
# double-counting that occurs when async_post_call_success_hook also increments
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_proxy_total_requests_metric"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_proxy_total_requests_metric,
"litellm_proxy_total_requests_metric",
enum_values,
label_context=label_context,
)
self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
def _increment_token_metrics(
self,
@ -1208,6 +1227,7 @@ class PrometheusLogger(CustomLogger):
user_api_team_alias: Optional[str],
user_id: Optional[str],
enum_values: UserAPIKeyLabelValues,
label_context: Optional[PrometheusLabelFactoryContext] = None,
):
verbose_logger.debug("prometheus Logging - Enters token metrics function")
# token metrics
@ -1217,41 +1237,36 @@ class PrometheusLogger(CustomLogger):
):
_tags = standard_logging_payload["request_tags"]
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_total_tokens_metric"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_tokens_metric,
"litellm_total_tokens_metric",
enum_values,
label_context=label_context,
amount=float(standard_logging_payload["total_tokens"]),
)
self.litellm_tokens_metric.labels(**_labels).inc(
standard_logging_payload["total_tokens"]
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_input_tokens_metric,
"litellm_input_tokens_metric",
enum_values,
label_context=label_context,
amount=float(standard_logging_payload["prompt_tokens"]),
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_input_tokens_metric"
),
enum_values=enum_values,
)
self.litellm_input_tokens_metric.labels(**_labels).inc(
standard_logging_payload["prompt_tokens"]
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_output_tokens_metric"
),
enum_values=enum_values,
)
self.litellm_output_tokens_metric.labels(**_labels).inc(
standard_logging_payload["completion_tokens"]
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_output_tokens_metric,
"litellm_output_tokens_metric",
enum_values,
label_context=label_context,
amount=float(standard_logging_payload["completion_tokens"]),
)
def _increment_cache_metrics(
self,
standard_logging_payload: StandardLoggingPayload,
enum_values: UserAPIKeyLabelValues,
label_context: Optional[PrometheusLabelFactoryContext] = None,
):
"""
Increment cache-related Prometheus metrics based on cache hit/miss status.
@ -1268,33 +1283,34 @@ class PrometheusLogger(CustomLogger):
if cache_hit is True:
# Increment cache hits counter
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_cache_hits_metric"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_cache_hits_metric,
"litellm_cache_hits_metric",
enum_values,
label_context=label_context,
)
self.litellm_cache_hits_metric.labels(**_labels).inc()
# Increment cached tokens counter
total_tokens = standard_logging_payload.get("total_tokens", 0)
if total_tokens > 0:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_cached_tokens_metric"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_cached_tokens_metric,
"litellm_cached_tokens_metric",
enum_values,
label_context=label_context,
amount=float(total_tokens),
)
self.litellm_cached_tokens_metric.labels(**_labels).inc(total_tokens)
else:
# cache_hit is False - increment cache misses counter
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_cache_misses_metric"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_cache_misses_metric,
"litellm_cache_misses_metric",
enum_values,
label_context=label_context,
)
self.litellm_cache_misses_metric.labels(**_labels).inc()
async def _increment_remaining_budget_metrics(
self,
@ -1361,25 +1377,24 @@ class PrometheusLogger(CustomLogger):
user_id: Optional[str],
response_cost: float,
enum_values: UserAPIKeyLabelValues,
label_context: Optional[PrometheusLabelFactoryContext] = None,
):
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_requests_metric"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_requests_metric,
"litellm_requests_metric",
enum_values,
label_context=label_context,
)
self.litellm_requests_metric.labels(**_labels).inc()
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_spend_metric"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_spend_metric,
"litellm_spend_metric",
enum_values,
label_context=label_context,
amount=float(response_cost),
)
self.litellm_spend_metric.labels(**_labels).inc(response_cost)
def _set_virtual_key_rate_limit_metrics(
self,
user_api_key: Optional[str],
@ -1430,6 +1445,7 @@ class PrometheusLogger(CustomLogger):
user_api_team: Optional[str],
user_api_team_alias: Optional[str],
enum_values: UserAPIKeyLabelValues,
label_context: Optional[PrometheusLabelFactoryContext] = None,
):
# latency metrics
end_time: datetime = kwargs.get("end_time") or datetime.now()
@ -1449,6 +1465,7 @@ class PrometheusLogger(CustomLogger):
metric_name="litellm_llm_api_time_to_first_token_metric"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_llm_api_time_to_first_token_metric.labels(
**_ttft_labels
@ -1468,6 +1485,7 @@ class PrometheusLogger(CustomLogger):
metric_name="litellm_llm_api_latency_metric"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_llm_api_latency_metric.labels(**_labels).observe(
api_call_total_time_seconds
@ -1484,6 +1502,7 @@ class PrometheusLogger(CustomLogger):
metric_name="litellm_request_total_latency_metric"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_request_total_latency_metric.labels(**_labels).observe(
total_time_seconds
@ -1500,16 +1519,16 @@ class PrometheusLogger(CustomLogger):
metric_name="litellm_request_queue_time_seconds"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_request_queue_time_metric.labels(**_labels).observe(
queue_time_seconds
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
from litellm.types.utils import StandardLoggingPayload
verbose_logger.debug(
f"prometheus Logging - Enters failure logging function for kwargs {kwargs}"
"prometheus Logging - Enters failure logging function (kwargs keys: %s)",
list(kwargs.keys()) if isinstance(kwargs, dict) else type(kwargs).__name__,
)
standard_logging_payload: StandardLoggingPayload = kwargs.get(
@ -1767,25 +1786,27 @@ class PrometheusLogger(CustomLogger):
client_ip=_metadata.get("requester_ip_address"),
user_agent=_metadata.get("user_agent"),
model_id=model_id,
stream=str(request_data.get("stream"))
if litellm.prometheus_emit_stream_label
else None,
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_proxy_failed_requests_metric"
stream=(
str(request_data.get("stream"))
if litellm.prometheus_emit_stream_label
else None
),
enum_values=enum_values,
)
self.litellm_proxy_failed_requests_metric.labels(**_labels).inc()
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_proxy_total_requests_metric"
),
enum_values=enum_values,
_label_ctx = PrometheusLabelFactoryContext(enum_values)
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_proxy_failed_requests_metric,
"litellm_proxy_failed_requests_metric",
enum_values,
label_context=_label_ctx,
)
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_proxy_total_requests_metric,
"litellm_proxy_total_requests_metric",
enum_values,
label_context=_label_ctx,
)
self.litellm_proxy_total_requests_metric.labels(**_labels).inc()
except Exception as e:
verbose_logger.exception(
@ -2015,22 +2036,23 @@ class PrometheusLogger(CustomLogger):
api_base=api_base,
api_provider=llm_provider or "",
)
_deployment_label_ctx = PrometheusLabelFactoryContext(enum_values)
if exception is not None:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_deployment_failure_responses"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_deployment_failure_responses,
"litellm_deployment_failure_responses",
enum_values,
label_context=_deployment_label_ctx,
)
self.litellm_deployment_failure_responses.labels(**_labels).inc()
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_deployment_total_requests"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_deployment_total_requests,
"litellm_deployment_total_requests",
enum_values,
label_context=_deployment_label_ctx,
)
self.litellm_deployment_total_requests.labels(**_labels).inc()
pass
except Exception as e:
@ -2090,12 +2112,13 @@ class PrometheusLogger(CustomLogger):
end_time,
enum_values: UserAPIKeyLabelValues,
output_tokens: float = 1.0,
label_context: Optional[PrometheusLabelFactoryContext] = None,
):
try:
verbose_logger.debug("setting remaining tokens requests metric")
standard_logging_payload: Optional[
StandardLoggingPayload
] = request_kwargs.get("standard_logging_object")
standard_logging_payload: Optional[StandardLoggingPayload] = (
request_kwargs.get("standard_logging_object")
)
if standard_logging_payload is None:
return
@ -2147,6 +2170,7 @@ class PrometheusLogger(CustomLogger):
metric_name="litellm_overhead_latency_metric"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_overhead_latency_metric.labels(**_labels).observe(
litellm_overhead_time_ms / 1000
@ -2164,6 +2188,7 @@ class PrometheusLogger(CustomLogger):
metric_name="litellm_remaining_requests_metric"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_remaining_requests_metric.labels(**_labels).set(
remaining_requests
@ -2175,6 +2200,7 @@ class PrometheusLogger(CustomLogger):
metric_name="litellm_remaining_tokens_metric"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_remaining_tokens_metric.labels(**_labels).set(
remaining_tokens
@ -2191,21 +2217,20 @@ class PrometheusLogger(CustomLogger):
api_provider=llm_provider or "",
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_deployment_success_responses"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_deployment_success_responses,
"litellm_deployment_success_responses",
enum_values,
label_context=label_context,
)
self.litellm_deployment_success_responses.labels(**_labels).inc()
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_deployment_total_requests"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_deployment_total_requests,
"litellm_deployment_total_requests",
enum_values,
label_context=label_context,
)
self.litellm_deployment_total_requests.labels(**_labels).inc()
# Track deployment Latency
response_ms: timedelta = end_time - start_time
@ -2235,6 +2260,7 @@ class PrometheusLogger(CustomLogger):
metric_name="litellm_deployment_latency_per_output_token"
),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_deployment_latency_per_output_token.labels(
**_labels
@ -2468,13 +2494,13 @@ class PrometheusLogger(CustomLogger):
exception_class=self._get_exception_class_name(original_exception),
tags=_tags,
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_deployment_successful_fallbacks"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_deployment_successful_fallbacks,
"litellm_deployment_successful_fallbacks",
enum_values,
label_context=PrometheusLabelFactoryContext(enum_values),
)
self.litellm_deployment_successful_fallbacks.labels(**_labels).inc()
async def log_failure_fallback_event(
self, original_model_group: str, kwargs: dict, original_exception: Exception
@ -2514,13 +2540,13 @@ class PrometheusLogger(CustomLogger):
tags=_tags,
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(
metric_name="litellm_deployment_failed_fallbacks"
),
enum_values=enum_values,
PrometheusLogger._inc_labeled_counter(
self,
self.litellm_deployment_failed_fallbacks,
"litellm_deployment_failed_fallbacks",
enum_values,
label_context=PrometheusLabelFactoryContext(enum_values),
)
self.litellm_deployment_failed_fallbacks.labels(**_labels).inc()
def set_litellm_deployment_state(
self,
@ -2638,7 +2664,7 @@ class PrometheusLogger(CustomLogger):
self,
data_fetch_function: Callable[..., Awaitable[Tuple[List[Any], Optional[int]]]],
set_metrics_function: Callable[[List[Any]], Awaitable[None]],
data_type: Literal["teams", "keys", "users"],
data_type: Literal["teams", "keys", "users", "orgs"],
):
"""
Generic method to initialize budget metrics for teams or API keys.
@ -2714,8 +2740,6 @@ class PrometheusLogger(CustomLogger):
"""
Initialize API key budget metrics by reusing the generic pagination logic.
"""
from typing import Union
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
from litellm.proxy.management_endpoints.key_management_endpoints import (
_list_key_helper,
@ -2728,9 +2752,7 @@ class PrometheusLogger(CustomLogger):
)
return
async def fetch_keys(
page_size: int, page: int
) -> Tuple[
async def fetch_keys(page_size: int, page: int) -> Tuple[
List[Union[str, UserAPIKeyAuth, LiteLLM_DeletedVerificationToken]],
Optional[int],
]:
@ -2762,7 +2784,6 @@ class PrometheusLogger(CustomLogger):
"""
Initialize user budget metrics by reusing the generic pagination logic.
"""
from litellm.proxy._types import LiteLLM_UserTable
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
@ -2921,9 +2942,11 @@ class PrometheusLogger(CustomLogger):
org_alias=org.organization_alias or "",
spend=org.spend or 0.0,
max_budget=budget_table.max_budget if budget_table else None,
budget_reset_at=getattr(budget_table, "budget_reset_at", None)
if budget_table
else None,
budget_reset_at=(
getattr(budget_table, "budget_reset_at", None)
if budget_table
else None
),
)
async def _set_team_budget_metrics_after_api_request(
@ -3403,12 +3426,11 @@ class PrometheusLogger(CustomLogger):
It emits the current remaining budget metrics for all Keys and Teams.
"""
from litellm.constants import PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES
from litellm.integrations.custom_logger import CustomLogger
prometheus_loggers: List[
CustomLogger
] = litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=PrometheusLogger
prometheus_loggers: List[CustomLogger] = (
litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=PrometheusLogger
)
)
# we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them
verbose_logger.debug("found %s prometheus loggers", len(prometheus_loggers))
@ -3458,16 +3480,58 @@ class PrometheusLogger(CustomLogger):
)
def _prometheus_labels_from_context(
supported_enum_labels: List[str],
ctx: PrometheusLabelFactoryContext,
) -> Dict[str, Optional[str]]:
filtered_labels: Dict[str, Optional[str]] = {
label: ctx._sanitized_enum[label]
for label in supported_enum_labels
if label in ctx._sanitized_enum
}
if UserAPIKeyLabelNames.END_USER.value in filtered_labels:
filtered_labels[UserAPIKeyLabelNames.END_USER.value] = (
ctx.get_resolved_end_user()
)
for sk, val in ctx._custom_by_sanitized_key.items():
if sk in supported_enum_labels:
filtered_labels[sk] = val
for k, v in ctx._tag_labels.items():
if k in supported_enum_labels:
filtered_labels[k] = v
for label in supported_enum_labels:
if label not in filtered_labels:
filtered_labels[label] = None
return filtered_labels
def prometheus_label_factory(
supported_enum_labels: List[str],
enum_values: UserAPIKeyLabelValues,
tag: Optional[str] = None,
*,
label_context: Optional[PrometheusLabelFactoryContext] = None,
) -> dict:
"""
Returns a dictionary of label + values for prometheus.
Ensures end_user param is not sent to prometheus if it is not supported.
When ``label_context`` is provided, it must have been built from the same
``enum_values`` object; work is amortized (single model_dump, tag map, etc.).
"""
if label_context is not None:
if label_context.enum_values is not enum_values:
raise ValueError(
"label_context.enum_values must be the same object as enum_values"
)
return _prometheus_labels_from_context(supported_enum_labels, label_context)
# Extract dictionary from Pydantic object
enum_dict = enum_values.model_dump()
@ -3541,7 +3605,7 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]:
def _tag_matches_wildcard_configured_pattern(
tags: List[str], configured_tag: str
tags: Sequence[str], configured_tag: str
) -> bool:
"""
Check if any of the request tags matches a wildcard configured pattern
@ -3573,7 +3637,7 @@ def _tag_matches_wildcard_configured_pattern(
return any(re.match(pattern=regex_pattern, string=tag) for tag in tags)
def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]:
def get_custom_labels_from_tags(tags: Sequence[str]) -> Dict[str, str]:
"""
Get custom labels from tags based on admin configuration.

80
litellm/integrations/prometheus_helpers/__init__.py Normal file
View file

@ -0,0 +1,80 @@
"""
Helpers for the Prometheus integration (extracted to keep ``prometheus.py`` smaller).
``PrometheusLabelFactoryContext`` lives here so it has a dedicated module.
"""
from __future__ import annotations
from typing import Any, Dict, Optional, cast
from litellm.types.integrations.prometheus import (
UserAPIKeyLabelValues,
_sanitize_prometheus_label_name,
_sanitize_prometheus_label_value,
)
_get_end_user_id_for_cost_tracking = None
def _get_cached_end_user_id_for_cost_tracking():
"""
Get cached get_end_user_id_for_cost_tracking function.
Lazy imports on first call to avoid loading utils.py at import time (60MB saved).
Subsequent calls use cached function for better performance.
"""
global _get_end_user_id_for_cost_tracking
if _get_end_user_id_for_cost_tracking is None:
from litellm.utils import get_end_user_id_for_cost_tracking
_get_end_user_id_for_cost_tracking = get_end_user_id_for_cost_tracking
return _get_end_user_id_for_cost_tracking
class PrometheusLabelFactoryContext:
"""
Precomputes per-request label inputs so prometheus_label_factory can subset
per metric without repeated model_dump / tag / metadata work.
"""
__slots__ = (
"enum_values",
"_sanitized_enum",
"_custom_by_sanitized_key",
"_tag_labels",
"_resolved_end_user",
)
_END_USER_NOT_COMPUTED = object()
def __init__(self, enum_values: UserAPIKeyLabelValues) -> None:
self.enum_values = enum_values
enum_dict = enum_values.model_dump()
self._sanitized_enum: Dict[str, Optional[str]] = {
k: _sanitize_prometheus_label_value(v) for k, v in enum_dict.items()
}
self._custom_by_sanitized_key: Dict[str, Optional[str]] = {}
if enum_values.custom_metadata_labels is not None:
for key, value in enum_values.custom_metadata_labels.items():
sk = _sanitize_prometheus_label_name(key)
self._custom_by_sanitized_key[sk] = _sanitize_prometheus_label_value(
value
)
self._tag_labels: Dict[str, Optional[str]] = {}
if enum_values.tags is not None:
# Late import avoids circular import: ``prometheus`` imports this module.
from litellm.integrations.prometheus import get_custom_labels_from_tags
for k, v in get_custom_labels_from_tags(enum_values.tags).items():
self._tag_labels[k] = _sanitize_prometheus_label_value(v)
# Use a dedicated sentinel so `None` can be cached as a computed result.
self._resolved_end_user: Any = self._END_USER_NOT_COMPUTED
def get_resolved_end_user(self) -> Optional[str]:
if self._resolved_end_user is self._END_USER_NOT_COMPUTED:
fn = _get_cached_end_user_id_for_cost_tracking()
self._resolved_end_user = fn(
litellm_params={"user_api_key_end_user_id": self.enum_values.end_user},
service_type="prometheus",
)
return cast(Optional[str], self._resolved_end_user)

4
litellm/integrations/prometheus_services.py
View file

@ -38,7 +38,9 @@ class PrometheusServicesLogger:
_custom_buckets = litellm.prometheus_latency_buckets
self.latency_buckets = (
tuple(_custom_buckets) if _custom_buckets is not None else LATENCY_BUCKETS
tuple(_custom_buckets)
if _custom_buckets is not None
else LATENCY_BUCKETS
)
self.Histogram = Histogram

8
litellm/integrations/s3_v2.py
View file

@ -597,9 +597,11 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
request_url = prepped.url or url
httpx_client = _get_httpx_client(
params={"ssl_verify": self.s3_verify}
if self.s3_verify is not None
else None
params=(
{"ssl_verify": self.s3_verify}
if self.s3_verify is not None
else None
)
)
# Make the request with retry for transient S3 errors (500/503)
max_retries = 3

16
litellm/integrations/vantage/vantage_logger.py
View file

@ -83,9 +83,11 @@ class VantageLogger(FocusLogger):
verbose_logger.debug(
"VantageLogger initialized (integration_token=%s)",
resolved_token[:4] + "***"
if resolved_token and len(resolved_token) > 4
else "***",
(
resolved_token[:4] + "***"
if resolved_token and len(resolved_token) > 4
else "***"
),
)
async def initialize_focus_export_job(self) -> None:
@ -124,10 +126,10 @@ class VantageLogger(FocusLogger):
scheduler: AsyncIOScheduler,
) -> None:
"""Register the Vantage export job with the provided scheduler."""
vantage_loggers: List[
CustomLogger
] = litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=VantageLogger
vantage_loggers: List[CustomLogger] = (
litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=VantageLogger
)
)
if not vantage_loggers:
verbose_logger.debug("No Vantage logger registered; skipping scheduler")

36
litellm/integrations/vector_store_integrations/vector_store_pre_call_hook.py
View file

@ -88,12 +88,12 @@ class VectorStorePreCallHook(CustomLogger):
pass
# Use database fallback to ensure synchronization across instances
vector_stores_to_run: List[
LiteLLM_ManagedVectorStore
] = await litellm.vector_store_registry.pop_vector_stores_to_run_with_db_fallback(
non_default_params=non_default_params,
tools=tools,
prisma_client=prisma_client,
vector_stores_to_run: List[LiteLLM_ManagedVectorStore] = (
await litellm.vector_store_registry.pop_vector_stores_to_run_with_db_fallback(
non_default_params=non_default_params,
tools=tools,
prisma_client=prisma_client,
)
)
if not vector_stores_to_run:
@ -147,9 +147,9 @@ class VectorStorePreCallHook(CustomLogger):
# Store search results as-is (already in OpenAI-compatible format)
if litellm_logging_obj and all_search_results:
litellm_logging_obj.model_call_details[
"search_results"
] = all_search_results
litellm_logging_obj.model_call_details["search_results"] = (
all_search_results
)
return model, modified_messages, non_default_params
@ -208,9 +208,9 @@ class VectorStorePreCallHook(CustomLogger):
Returns:
Modified list of messages with context appended
"""
search_response_data: Optional[
List[VectorStoreSearchResult]
] = search_response.get("data")
search_response_data: Optional[List[VectorStoreSearchResult]] = (
search_response.get("data")
)
if not search_response_data:
return messages
@ -268,9 +268,9 @@ class VectorStorePreCallHook(CustomLogger):
)
# Get search results from model_call_details (already in OpenAI format)
search_results: Optional[
List[VectorStoreSearchResponse]
] = litellm_logging_obj.model_call_details.get("search_results")
search_results: Optional[List[VectorStoreSearchResponse]] = (
litellm_logging_obj.model_call_details.get("search_results")
)
verbose_logger.debug(f"Search results found: {search_results is not None}")
@ -328,9 +328,9 @@ class VectorStorePreCallHook(CustomLogger):
)
# Get search results from model_call_details (already in OpenAI format)
search_results: Optional[
List[VectorStoreSearchResponse]
] = request_data.get("search_results")
search_results: Optional[List[VectorStoreSearchResponse]] = (
request_data.get("search_results")
)
verbose_logger.debug(
f"Search results found for streaming chunk: {search_results is not None}"

336
litellm/integrations/websearch_interception/handler.py
View file

@ -28,6 +28,10 @@ from litellm.integrations.websearch_interception.transformation import (
from litellm.types.integrations.websearch_interception import (
WebSearchInterceptionConfig,
)
from litellm.types.integrations.custom_logger import (
AgenticLoopPlan,
AgenticLoopRequestPatch,
)
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
@ -573,6 +577,35 @@ class WebSearchInterceptionLogger(CustomLogger):
kwargs=kwargs,
)
async def async_build_agentic_loop_plan(
self,
tools: Dict,
model: str,
messages: List[Dict],
response: Any,
anthropic_messages_provider_config: Any,
anthropic_messages_optional_request_params: Dict,
logging_obj: Any,
stream: bool,
kwargs: Dict,
) -> AgenticLoopPlan:
tool_calls = tools["tool_calls"]
thinking_blocks = tools.get("thinking_blocks", [])
request_patch = await self._build_anthropic_request_patch(
model=model,
messages=messages,
tool_calls=tool_calls,
thinking_blocks=thinking_blocks,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
logging_obj=logging_obj,
kwargs=kwargs,
)
return AgenticLoopPlan(
run_agentic_loop=True,
request_patch=request_patch,
metadata={"tool_type": "websearch", "response_format": "anthropic"},
)
async def async_run_chat_completion_agentic_loop(
self,
tools: Dict,
@ -608,6 +641,33 @@ class WebSearchInterceptionLogger(CustomLogger):
response_format=response_format,
)
async def async_build_chat_completion_agentic_loop_plan(
self,
tools: Dict,
model: str,
messages: List[Dict],
response: Any,
optional_params: Dict,
logging_obj: Any,
stream: bool,
kwargs: Dict,
) -> AgenticLoopPlan:
tool_calls = tools["tool_calls"]
response_format = tools.get("response_format", "openai")
request_patch = await self._build_chat_completion_request_patch(
model=model,
messages=messages,
tool_calls=tool_calls,
optional_params=optional_params,
kwargs=kwargs,
response_format=response_format,
)
return AgenticLoopPlan(
run_agentic_loop=True,
request_patch=request_patch,
metadata={"tool_type": "websearch", "response_format": response_format},
)
@staticmethod
def _resolve_max_tokens(
optional_params: Dict,
@ -672,7 +732,48 @@ class WebSearchInterceptionLogger(CustomLogger):
stream: bool,
kwargs: Dict,
) -> Any:
"""Execute litellm.search() and make follow-up request"""
"""Legacy path: execute search + build patch + run follow-up call."""
request_patch = await self._build_anthropic_request_patch(
model=model,
messages=messages,
tool_calls=tool_calls,
thinking_blocks=thinking_blocks,
anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
logging_obj=logging_obj,
kwargs=kwargs,
)
if request_patch.messages is None:
raise ValueError("WebSearchInterception: missing follow-up messages")
optional_params = dict(anthropic_messages_optional_request_params)
optional_params.update(request_patch.optional_params)
max_tokens = request_patch.max_tokens
if max_tokens is None:
max_tokens = cast(Optional[int], optional_params.pop("max_tokens", None))
else:
optional_params.pop("max_tokens", None)
if max_tokens is None:
max_tokens = cast(int, kwargs.get("max_tokens", 1024))
return await anthropic_messages.acreate(
max_tokens=max_tokens,
messages=request_patch.messages,
model=request_patch.model or model,
**optional_params,
**request_patch.kwargs,
)
async def _build_anthropic_request_patch(
self,
model: str,
messages: List[Dict],
tool_calls: List[Dict],
thinking_blocks: List[Dict],
anthropic_messages_optional_request_params: Dict,
logging_obj: Any,
kwargs: Dict,
) -> AgenticLoopRequestPatch:
"""Execute litellm.search() and build follow-up request patch."""
# Extract search queries from tool_use blocks
search_tasks = []
@ -721,20 +822,8 @@ class WebSearchInterceptionLogger(CustomLogger):
thinking_blocks=thinking_blocks,
)
# Make follow-up request with search results
# Type cast: user_message is a Dict for Anthropic format (default response_format)
follow_up_messages = messages + [assistant_message, cast(Dict, user_message)]
verbose_logger.debug(
"WebSearchInterception: Making follow-up request with search results"
)
verbose_logger.debug(
f"WebSearchInterception: Follow-up messages count: {len(follow_up_messages)}"
)
verbose_logger.debug(
f"WebSearchInterception: Last message (tool_result): {user_message}"
)
# Correlation context for structured logging
_call_id = getattr(logging_obj, "litellm_call_id", None) or kwargs.get(
"litellm_call_id", "unknown"
@ -742,61 +831,41 @@ class WebSearchInterceptionLogger(CustomLogger):
full_model_name = model # safe default before try block
# Use anthropic_messages.acreate for follow-up request
try:
max_tokens = self._resolve_max_tokens(
anthropic_messages_optional_request_params, kwargs
)
max_tokens = self._resolve_max_tokens(
anthropic_messages_optional_request_params, kwargs
)
verbose_logger.debug(
f"WebSearchInterception: Using max_tokens={max_tokens} for follow-up request"
)
verbose_logger.debug(
f"WebSearchInterception: Using max_tokens={max_tokens} for follow-up request"
)
# Create a copy of optional params without max_tokens (since we pass it explicitly)
optional_params_without_max_tokens = {
k: v
for k, v in anthropic_messages_optional_request_params.items()
if k != "max_tokens"
}
optional_params_without_max_tokens = {
k: v
for k, v in anthropic_messages_optional_request_params.items()
if k != "max_tokens"
}
kwargs_for_followup = self._prepare_followup_kwargs(kwargs)
kwargs_for_followup = self._prepare_followup_kwargs(kwargs)
# Get model from logging_obj.model_call_details["agentic_loop_params"]
# This preserves the full model name with provider prefix (e.g., "bedrock/invoke/...")
if logging_obj is not None:
agentic_params = logging_obj.model_call_details.get(
"agentic_loop_params", {}
)
full_model_name = agentic_params.get("model", model)
verbose_logger.debug(
f"WebSearchInterception: Using model name: {full_model_name}"
if logging_obj is not None:
agentic_params = logging_obj.model_call_details.get(
"agentic_loop_params", {}
)
final_response = await anthropic_messages.acreate(
max_tokens=max_tokens,
messages=follow_up_messages,
model=full_model_name,
**optional_params_without_max_tokens,
**kwargs_for_followup,
)
verbose_logger.debug(
f"WebSearchInterception: Follow-up request completed, response type: {type(final_response)}"
)
verbose_logger.debug(
f"WebSearchInterception: Final response: {final_response}"
)
return final_response
except Exception as e:
verbose_logger.exception(
"WebSearchInterception: Follow-up request failed "
"[call_id=%s model=%s messages=%d searches=%d]: %s",
_call_id,
full_model_name,
len(follow_up_messages),
len(final_search_results),
str(e),
)
raise
full_model_name = agentic_params.get("model", model)
verbose_logger.debug(
"WebSearchInterception: Built anthropic request patch "
"[call_id=%s model=%s messages=%d searches=%d]",
_call_id,
full_model_name,
len(follow_up_messages),
len(final_search_results),
)
return AgenticLoopRequestPatch(
model=full_model_name,
messages=follow_up_messages,
max_tokens=max_tokens,
optional_params=optional_params_without_max_tokens,
kwargs=kwargs_for_followup,
)
async def _execute_search(self, query: str) -> str:
"""Execute a single web search using router's search tools"""
@ -883,7 +952,36 @@ class WebSearchInterceptionLogger(CustomLogger):
kwargs: Dict,
response_format: str = "openai",
) -> Any:
"""Execute litellm.search() and make follow-up chat completion request"""
"""Legacy path: execute search + build patch + run follow-up call."""
request_patch = await self._build_chat_completion_request_patch(
model=model,
messages=messages,
tool_calls=tool_calls,
optional_params=optional_params,
kwargs=kwargs,
response_format=response_format,
)
if request_patch.messages is None:
raise ValueError("WebSearchInterception: missing follow-up messages")
params = dict(optional_params)
params.update(request_patch.optional_params)
return await litellm.acompletion(
model=request_patch.model or model,
messages=request_patch.messages,
**params,
**request_patch.kwargs,
)
async def _build_chat_completion_request_patch( # noqa: PLR0915
self,
model: str,
messages: List[Dict],
tool_calls: List[Dict],
optional_params: Dict,
kwargs: Dict,
response_format: str = "openai",
) -> AgenticLoopRequestPatch:
"""Execute litellm.search() and build chat-completion rerun patch."""
# Extract search queries from tool_calls
search_tasks = []
@ -963,74 +1061,56 @@ class WebSearchInterceptionLogger(CustomLogger):
f"WebSearchInterception: Follow-up messages count: {len(follow_up_messages)}"
)
# Use litellm.acompletion for follow-up request
try:
# Remove internal parameters that shouldn't be passed to follow-up request
internal_params = {
"_websearch_interception",
"acompletion",
"litellm_logging_obj",
"custom_llm_provider",
# Remove internal parameters that shouldn't be passed to follow-up request
internal_params = {
"_websearch_interception",
"acompletion",
"litellm_logging_obj",
"custom_llm_provider",
"model_alias_map",
"stream_response",
"custom_prompt_dict",
}
kwargs_for_followup = {
k: v
for k, v in kwargs.items()
if not k.startswith("_websearch_interception") and k not in internal_params
}
full_model_name = model
if "custom_llm_provider" in kwargs:
custom_llm_provider = kwargs["custom_llm_provider"]
if not model.startswith(custom_llm_provider) and "/" not in model:
full_model_name = f"{custom_llm_provider}/{model}"
verbose_logger.debug(
"WebSearchInterception: Built chat completion request patch model=%s messages=%d",
full_model_name,
len(follow_up_messages),
)
tools_param = optional_params.get("tools")
optional_params_clean = {
k: v
for k, v in optional_params.items()
if k
not in {
"tools",
"extra_body",
"model_alias_map",
"stream_response",
"custom_prompt_dict",
}
kwargs_for_followup = {
k: v
for k, v in kwargs.items()
if not k.startswith("_websearch_interception")
and k not in internal_params
}
}
if tools_param is not None:
optional_params_clean["tools"] = tools_param
# Get full model name from kwargs
full_model_name = model
if "custom_llm_provider" in kwargs:
custom_llm_provider = kwargs["custom_llm_provider"]
# Reconstruct full model name with provider prefix if needed
if not model.startswith(custom_llm_provider):
# Check if model already has a provider prefix
if "/" not in model:
full_model_name = f"{custom_llm_provider}/{model}"
verbose_logger.debug(
f"WebSearchInterception: Using model name: {full_model_name}"
)
# Prepare tools for follow-up request (same as original)
tools_param = optional_params.get("tools")
# Remove tools and extra_body from optional_params to avoid issues
# extra_body often contains internal LiteLLM params that shouldn't be forwarded
optional_params_clean = {
k: v
for k, v in optional_params.items()
if k
not in {
"tools",
"extra_body",
"model_alias_map",
"stream_response",
"custom_prompt_dict",
}
}
final_response = await litellm.acompletion(
model=full_model_name,
messages=follow_up_messages,
tools=tools_param,
**optional_params_clean,
**kwargs_for_followup,
)
verbose_logger.debug(
f"WebSearchInterception: Follow-up request completed, response type: {type(final_response)}"
)
return final_response
except Exception as e:
verbose_logger.exception(
f"WebSearchInterception: Follow-up request failed: {str(e)}"
)
raise
return AgenticLoopRequestPatch(
model=full_model_name,
messages=follow_up_messages,
optional_params=optional_params_clean,
kwargs=kwargs_for_followup,
)
async def _create_empty_search_result(self) -> str:
"""Create an empty search result for tool calls without queries"""

9
litellm/integrations/websearch_interception/transformation.py
View file

@ -3,6 +3,7 @@ WebSearch Tool Transformation
Transforms between Anthropic/OpenAI tool_use format and LiteLLM search format.
"""
import json
from typing import Any, Dict, List, Optional, Tuple, Union
@ -326,9 +327,11 @@ class WebSearchTransformation:
"type": "function",
"function": {
"name": tc["name"],
"arguments": json.dumps(tc["input"])
if isinstance(tc["input"], dict)
else str(tc["input"]),
"arguments": (
json.dumps(tc["input"])
if isinstance(tc["input"], dict)
else str(tc["input"])
),
},
}
for tc in tool_calls

3
litellm/integrations/weights_biases.py
View file

@ -21,8 +21,7 @@ try:
# contains a (known) object attribute
object: Literal["chat.completion", "edit", "text_completion"]
def __getitem__(self, key: K) -> V:
... # noqa
def __getitem__(self, key: K) -> V: ... # noqa
def get(self, key: K, default: Optional[V] = None) -> Optional[V]: # noqa
... # pragma: no cover

8
litellm/interactions/litellm_responses_transformation/transformation.py
View file

@ -45,10 +45,10 @@ class LiteLLMResponsesInteractionsConfig:
# Transform input
if input is not None:
responses_request[
"input"
] = LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
input
responses_request["input"] = (
LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input(
input
)
)
# Transform system_instruction -> instructions

6
litellm/litellm_core_utils/default_encoding.py
View file

@ -26,9 +26,9 @@ if custom_cache_dir:
else:
cache_dir = filename
os.environ[
"TIKTOKEN_CACHE_DIR"
] = cache_dir # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071
os.environ["TIKTOKEN_CACHE_DIR"] = (
cache_dir # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071
)
import tiktoken
import time

4
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -2460,7 +2460,9 @@ def exception_type( # type: ignore # noqa: PLR0915
setattr(e, "litellm_response_headers", litellm_response_headers)
raise e # it's already mapped
raised_exc = APIConnectionError(
message="{}\n{}".format(original_exception, _redact_string(traceback.format_exc())),
message="{}\n{}".format(
original_exception, _redact_string(traceback.format_exc())
),
llm_provider="",
model="",
)

Some files were not shown because too many files have changed in this diff Show more