diff --git a/.circleci/config.yml b/.circleci/config.yml
index eed94b77b93..c5260165877 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1670,8 +1670,9 @@ jobs:
name: Run proxy tests
command: |
prisma generate
- python -m pytest tests/test_litellm/proxy --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy.xml --durations=10 -n 16 --maxfail=5 --timeout=300 -vv --log-cli-level=WARNING
- no_output_timeout: 120m
+ export PYTHONUNBUFFERED=1
+ python -m pytest tests/test_litellm/proxy --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy.xml --durations=10 -n 8 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
+ no_output_timeout: 60m
- run:
name: Rename the coverage files
command: |
@@ -3597,6 +3598,7 @@ jobs:
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
+ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME="us-east-1" \
diff --git a/litellm-proxy-extras/build_and_publish.md b/litellm-proxy-extras/build_and_publish.md
new file mode 100644
index 00000000000..e283166200f
--- /dev/null
+++ b/litellm-proxy-extras/build_and_publish.md
@@ -0,0 +1,108 @@
+# Build & Publish `litellm-proxy-extras`
+
+This runbook covers building and publishing a new version of the `litellm-proxy-extras` PyPI package. For use by litellm engineers only.
+
+## Prerequisites
+
+- All `schema.prisma` files are in sync (see [migration_runbook.md](./migration_runbook.md) Step 0)
+- Migration has been generated and committed
+- You are in the `litellm-proxy-extras/` directory
+
+## Step 1: Bump the Version
+
+Update the version in `pyproject.toml`:
+
+```bash
+cd litellm-proxy-extras
+
+# Check current version
+grep 'version' pyproject.toml
+```
+
+Edit `pyproject.toml` and bump the version (both `[tool.poetry].version` and `[tool.commitizen].version`).
+
+## Step 2: Update Version in Root Package Files
+
+After bumping the version in `litellm-proxy-extras/pyproject.toml`, you **must** also update the version reference in the root-level files:
+
+| File | Line to update |
+|------|---------------|
+| `requirements.txt` | `litellm-proxy-extras==X.Y.Z` |
+| `pyproject.toml` (root) | `litellm-proxy-extras = {version = "X.Y.Z", optional = true}` |
+
+```bash
+# From the repo root — replace OLD with NEW version
+sed -i '' 's/litellm-proxy-extras==OLD/litellm-proxy-extras==NEW/' requirements.txt
+sed -i '' 's/litellm-proxy-extras = {version = "OLD"/litellm-proxy-extras = {version = "NEW"/' pyproject.toml
+```
+
+> **Do NOT skip this step.** The main `litellm` package pins the extras version — if you don't update these, users will install the old version.
+
+## Step 3: Install Build Dependencies
+
+```bash
+pip install build twine
+```
+
+## Step 4: Clean Old Artifacts
+
+```bash
+rm -rf dist/ build/ *.egg-info
+```
+
+## Step 5: Build the Package
+
+```bash
+python3 -m build
+```
+
+This creates `.tar.gz` and `.whl` files in the `dist/` directory.
+
+Verify the build output:
+
+```bash
+ls -la dist/
+```
+
+## Step 6: Upload to PyPI
+
+```bash
+twine upload dist/*
+```
+
+You will be prompted for your PyPI API token:
+
+```
+Enter your API token: pypi-...
+```
+
+> Use `__token__` as the username and your PyPI API token as the password.
+
+## Quick Reference (Copy-Paste)
+
+```bash
+cd litellm-proxy-extras
+rm -rf dist/ build/ *.egg-info
+python3 -m build
+twine upload dist/*
+```
+
+---
+
+## Do you want to build and publish a new `litellm-proxy-extras` package? (y/n)
+
+If **yes**, run the following commands in order:
+
+```bash
+cd litellm-proxy-extras
+pip install build twine
+rm -rf dist/ build/ *.egg-info
+python3 -m build
+twine upload dist/*
+```
+
+When `twine upload` runs, enter your PyPI credentials:
+- **Username:** `__token__`
+- **Password:** *(paste your PyPI API key)*
+
+If **no**, you're done — no package publish needed.
diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260214094754_schema_sync/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260214094754_schema_sync/migration.sql
new file mode 100644
index 00000000000..b5d5b978580
--- /dev/null
+++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260214094754_schema_sync/migration.sql
@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_GuardrailsTable" ADD COLUMN "team_id" TEXT;
+
diff --git a/litellm-proxy-extras/migration_runbook.md b/litellm-proxy-extras/migration_runbook.md
index 93948f24b13..3310b1626a8 100644
--- a/litellm-proxy-extras/migration_runbook.md
+++ b/litellm-proxy-extras/migration_runbook.md
@@ -2,7 +2,35 @@
This is a runbook for creating and running database migrations for the LiteLLM proxy. For use for litellm engineers only.
-## Quick Start
+## Step 0: Sync All `schema.prisma` Files
+
+Before doing anything else, make sure all `schema.prisma` files in the repo are in sync. There are multiple copies that must match:
+
+| File | Purpose |
+|------|---------|
+| `schema.prisma` (repo root) | Source of truth |
+| `litellm/proxy/schema.prisma` | Used by the proxy server |
+| `litellm-proxy-extras/litellm_proxy_extras/schema.prisma` | Used for migration generation |
+
+**Sync process:**
+
+```bash
+# 1. Diff all schema files against the root source of truth
+diff schema.prisma litellm/proxy/schema.prisma
+diff schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+
+# 2. If there are differences, copy the root schema to all locations
+cp schema.prisma litellm/proxy/schema.prisma
+cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+
+# 3. Verify all files are now identical
+diff schema.prisma litellm/proxy/schema.prisma && echo "proxy schema in sync" || echo "MISMATCH"
+diff schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma && echo "extras schema in sync" || echo "MISMATCH"
+```
+
+> **Do NOT proceed to migration generation until all schema files are identical.**
+
+## Step 1: Quick Start — Generate Migration
```bash
# Install deps (one time)
@@ -43,8 +71,13 @@ rm -rf litellm-proxy-extras/litellm_proxy_extras/migrations/[empty_dir]
## Rules
-- Update `schema.prisma` first
+- Sync all `schema.prisma` files first (Step 0)
+- Update `schema.prisma` at the repo root first, then sync copies
- Review generated SQL before committing
- Use descriptive migration names
- Never edit existing migration files
- Commit schema + migration together
+
+---
+
+**Done with migration?** See [build_and_publish.md](./build_and_publish.md) to publish a new `litellm-proxy-extras` package.
diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml
index 909c5e7e6cd..45a6f975c78 100644
--- a/litellm-proxy-extras/pyproject.toml
+++ b/litellm-proxy-extras/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
-version = "0.4.37"
+version = "0.4.38"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
-version = "0.4.37"
+version = "0.4.38"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",
diff --git a/litellm/constants.py b/litellm/constants.py
index 650896e7437..03f80a8cb78 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -106,9 +106,7 @@ MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL = int(
# npm/npx needs a writable cache dir; in containers the default (~/.npm)
# may not exist or be read-only. /tmp is always writable.
MCP_NPM_CACHE_DIR = os.getenv("MCP_NPM_CACHE_DIR", "/tmp/.npm_mcp_cache")
-MCP_OAUTH2_TOKEN_CACHE_MIN_TTL = int(
- os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10")
-)
+MCP_OAUTH2_TOKEN_CACHE_MIN_TTL = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10"))
LITELLM_UI_ALLOW_HEADERS = [
"x-litellm-semantic-filter",
@@ -131,7 +129,7 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int(
# Maximum number of callbacks that can be registered
# This prevents callbacks from exponentially growing and consuming CPU resources
# Override with LITELLM_MAX_CALLBACKS env var for large deployments (e.g., many teams with guardrails)
-MAX_CALLBACKS = get_env_int("LITELLM_MAX_CALLBACKS", 30)
+MAX_CALLBACKS = get_env_int("LITELLM_MAX_CALLBACKS", 100)
# Generic fallback for unknown models
DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET = int(
@@ -167,15 +165,19 @@ _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client fo
# Aiohttp connection pooling - prevents memory leaks from unbounded connection growth
# Set to 0 for unlimited (not recommended for production)
AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 300))
-AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 50))
+AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int(
+ os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 50)
+)
AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120))
AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300))
# enable_cleanup_closed is only needed for Python versions with the SSL leak bug
# Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960)
# Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78
-AIOHTTP_NEEDS_CLEANUP_CLOSED = (
- (3, 13, 0) <= sys.version_info < (3, 13, 1) or sys.version_info < (3, 12, 7)
-)
+AIOHTTP_NEEDS_CLEANUP_CLOSED = (3, 13, 0) <= sys.version_info < (
+ 3,
+ 13,
+ 1,
+) or sys.version_info < (3, 12, 7)
# WebSocket constants
# Default to None (unlimited) to match OpenAI's official agents SDK behavior
@@ -213,15 +215,15 @@ REDIS_UPDATE_BUFFER_KEY = "litellm_spend_update_buffer"
REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_spend_update_buffer"
REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_team_spend_update_buffer"
REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_org_spend_update_buffer"
-REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_end_user_spend_update_buffer"
+REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY = (
+ "litellm_daily_end_user_spend_update_buffer"
+)
REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_agent_spend_update_buffer"
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_tag_spend_update_buffer"
MAX_REDIS_BUFFER_DEQUEUE_COUNT = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100))
MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", 2000))
# Bounds asyncio.Queue() instances (log queues, spend update queues, etc.) to prevent unbounded memory growth
-LITELLM_ASYNCIO_QUEUE_MAXSIZE = int(
- os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000)
-)
+LITELLM_ASYNCIO_QUEUE_MAXSIZE = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000))
MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = int(
os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000)
)
@@ -343,7 +345,9 @@ MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000))
#### Networking settings ####
request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds
-DEFAULT_A2A_AGENT_TIMEOUT: float = float(os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)) # 10 minutes
+DEFAULT_A2A_AGENT_TIMEOUT: float = float(
+ os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)
+) # 10 minutes
# Patterns that indicate a localhost/internal URL in A2A agent cards that should be
# replaced with the original base_url. This is a common misconfiguration where
# developers deploy agents with development URLs in their agent cards.
@@ -395,8 +399,12 @@ DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE = os.getenv(
"DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE", "streaming.chunk.yield"
)
-EMAIL_BUDGET_ALERT_TTL = int(os.getenv("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60)) # 24 hours in seconds
-EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE = float(os.getenv("EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE", 0.8)) # 80% of max budget
+EMAIL_BUDGET_ALERT_TTL = int(
+ os.getenv("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60)
+) # 24 hours in seconds
+EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE = float(
+ os.getenv("EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE", 0.8)
+) # 80% of max budget
############### LLM Provider Constants ###############
### ANTHROPIC CONSTANTS ###
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION = os.getenv(
@@ -1150,7 +1158,17 @@ known_tokenizer_config = {
}
-OPENAI_FINISH_REASONS = ["stop", "length", "function_call", "content_filter", "null", "finish_reason_unspecified", "malformed_function_call", "guardrail_intervened", "eos"]
+OPENAI_FINISH_REASONS = [
+ "stop",
+ "length",
+ "function_call",
+ "content_filter",
+ "null",
+ "finish_reason_unspecified",
+ "malformed_function_call",
+ "guardrail_intervened",
+ "eos",
+]
HUMANLOOP_PROMPT_CACHE_TTL_SECONDS = int(
os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60)
) # 1 minute
@@ -1250,8 +1268,8 @@ CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session"
CLI_JWT_TOKEN_NAME = "cli-jwt-token"
# Support both CLI_JWT_EXPIRATION_HOURS and LITELLM_CLI_JWT_EXPIRATION_HOURS for backwards compatibility
CLI_JWT_EXPIRATION_HOURS = int(
- os.getenv("CLI_JWT_EXPIRATION_HOURS")
- or os.getenv("LITELLM_CLI_JWT_EXPIRATION_HOURS")
+ os.getenv("CLI_JWT_EXPIRATION_HOURS")
+ or os.getenv("LITELLM_CLI_JWT_EXPIRATION_HOURS")
or 24
)
@@ -1435,9 +1453,7 @@ MICROSOFT_USER_EMAIL_ATTRIBUTE = str(
MICROSOFT_USER_DISPLAY_NAME_ATTRIBUTE = str(
os.getenv("MICROSOFT_USER_DISPLAY_NAME_ATTRIBUTE", "displayName")
)
-MICROSOFT_USER_ID_ATTRIBUTE = str(
- os.getenv("MICROSOFT_USER_ID_ATTRIBUTE", "id")
-)
+MICROSOFT_USER_ID_ATTRIBUTE = str(os.getenv("MICROSOFT_USER_ID_ATTRIBUTE", "id"))
MICROSOFT_USER_FIRST_NAME_ATTRIBUTE = str(
os.getenv("MICROSOFT_USER_FIRST_NAME_ATTRIBUTE", "givenName")
)
diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py
index 4ea22dbd90f..fe082843306 100644
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@@ -74,6 +74,7 @@ from litellm.llms.vertex_ai.cost_calculator import (
from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_router
from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token
from litellm.responses.utils import ResponseAPILoggingUtils
+from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.llms.openai import (
HttpxBinaryResponseContent,
ImageGenerationRequestQuality,
@@ -150,32 +151,33 @@ def _get_additional_costs(
) -> Optional[dict]:
"""
Calculate additional costs beyond standard token costs.
-
+
This function delegates to provider-specific config classes to calculate
any additional costs like routing fees, infrastructure costs, etc.
-
+
Args:
model: The model name
custom_llm_provider: The provider name (optional)
prompt_tokens: Number of prompt tokens
completion_tokens: Number of completion tokens
-
+
Returns:
Optional dictionary with cost names and amounts, or None if no additional costs
"""
if not custom_llm_provider:
return None
-
+
try:
config_class = None
if custom_llm_provider == "azure_ai":
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+
config_class = AzureFoundryModelInfo.get_azure_ai_config_for_model(model)
# Add more providers here as needed
# elif custom_llm_provider == "other_provider":
# config_class = get_other_provider_config(model)
-
- if config_class and hasattr(config_class, 'calculate_additional_costs'):
+
+ if config_class and hasattr(config_class, "calculate_additional_costs"):
return config_class.calculate_additional_costs(
model=model,
prompt_tokens=prompt_tokens,
@@ -183,7 +185,7 @@ def _get_additional_costs(
)
except Exception as e:
verbose_logger.debug(f"Error calculating additional costs: {e}")
-
+
return None
@@ -748,6 +750,8 @@ def _infer_call_type(
return "image_generation"
elif isinstance(completion_response, TextCompletionResponse):
return "text_completion"
+ elif isinstance(completion_response, LiteLLMSendMessageResponse):
+ return "send_message"
return call_type
@@ -1037,9 +1041,9 @@ def completion_cost( # noqa: PLR0915
or isinstance(completion_response, dict)
): # tts returns a custom class
if isinstance(completion_response, dict):
- usage_obj: Optional[
- Union[dict, Usage]
- ] = completion_response.get("usage", {})
+ usage_obj: Optional[Union[dict, Usage]] = (
+ completion_response.get("usage", {})
+ )
else:
usage_obj = getattr(completion_response, "usage", {})
if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects(
@@ -1393,7 +1397,7 @@ def completion_cost( # noqa: PLR0915
service_tier=service_tier,
response=completion_response,
)
-
+
# Get additional costs from provider (e.g., routing fees, infrastructure costs)
additional_costs = _get_additional_costs(
model=model,
@@ -1401,7 +1405,7 @@ def completion_cost( # noqa: PLR0915
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
)
-
+
_final_cost = (
prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar
)
diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py
index 8955d3619f7..b96ec72b04e 100644
--- a/litellm/integrations/langfuse/langfuse_otel.py
+++ b/litellm/integrations/langfuse/langfuse_otel.py
@@ -1,6 +1,7 @@
import base64
import json # <--- NEW
import os
+from datetime import datetime
from typing import TYPE_CHECKING, Any, Optional, Union
from litellm._logging import verbose_logger
@@ -392,6 +393,22 @@ class LangfuseOtelLogger(OpenTelemetry):
return dynamic_headers
+ def create_litellm_proxy_request_started_span(
+ self,
+ start_time: datetime,
+ headers: dict,
+ ) -> Optional[Span]:
+ """
+ Override to prevent creating empty proxy request spans.
+
+ Langfuse should only receive spans for actual LLM calls, not for
+ internal proxy operations (auth, postgres, proxy_pre_call, etc.).
+
+ By returning None, we prevent the parent span from being created,
+ which in turn prevents empty traces from being sent to Langfuse.
+ """
+ return None
+
async def async_service_success_hook(self, *args, **kwargs):
"""
Langfuse should not receive service success logs.
diff --git a/litellm/litellm_core_utils/api_route_to_call_types.py b/litellm/litellm_core_utils/api_route_to_call_types.py
index 4146ff6d6a6..2ae9986ce94 100644
--- a/litellm/litellm_core_utils/api_route_to_call_types.py
+++ b/litellm/litellm_core_utils/api_route_to_call_types.py
@@ -3,6 +3,9 @@ Dictionary mapping API routes to their corresponding CallTypes in LiteLLM.
This dictionary maps each API endpoint to the CallTypes that can be used for that route.
Each route can have both async (prefixed with 'a') and sync call types.
+
+Route patterns may contain placeholders like {agent_id}, {model}, {batch_id}; these
+match a single path segment when resolving call types for a concrete path.
"""
from typing import List, Optional
@@ -10,17 +13,43 @@ from typing import List, Optional
from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes
+def _route_matches_pattern(route: str, pattern: str) -> bool:
+ """
+ Return True if the concrete route matches the pattern.
+ Pattern segments like {param} match any single path segment.
+ """
+ route_parts = route.strip("/").split("/")
+ pattern_parts = pattern.strip("/").split("/")
+ if len(route_parts) != len(pattern_parts):
+ return False
+ for r, p in zip(route_parts, pattern_parts):
+ if p.startswith("{") and p.endswith("}"):
+ continue
+ if r != p:
+ return False
+ return True
+
+
def get_call_types_for_route(route: str) -> Optional[List[CallTypes]]:
"""
Get the list of CallTypes for a given API route.
+ Supports both exact keys and dynamic patterns (e.g. /a2a/my-agent/message/send
+ matches /a2a/{agent_id}/message/send).
+
Args:
- route: API route path (e.g., "/chat/completions")
+ route: API route path (e.g., "/chat/completions" or "/a2a/my-pydantic-agent/message/send")
Returns:
List of CallTypes for that route, or None if route not found
"""
- return API_ROUTE_TO_CALL_TYPES.get(route, None)
+ exact = API_ROUTE_TO_CALL_TYPES.get(route, None)
+ if exact is not None:
+ return exact
+ for pattern, call_types in API_ROUTE_TO_CALL_TYPES.items():
+ if _route_matches_pattern(route, pattern):
+ return call_types
+ return None
def get_routes_for_call_type(call_type: CallTypes) -> list:
diff --git a/litellm/llms/a2a/chat/guardrail_translation/handler.py b/litellm/llms/a2a/chat/guardrail_translation/handler.py
index 770453f2def..fbd1da749c2 100644
--- a/litellm/llms/a2a/chat/guardrail_translation/handler.py
+++ b/litellm/llms/a2a/chat/guardrail_translation/handler.py
@@ -10,6 +10,7 @@ A2A Protocol Format:
- Output: JSON-RPC 2.0 with result containing message/artifact parts
"""
+import json
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from litellm._logging import verbose_proxy_logger
@@ -206,6 +207,118 @@ class A2AGuardrailHandler(BaseTranslation):
response["result"] = result
return response
+ async def process_output_streaming_response(
+ self,
+ responses_so_far: List[Any],
+ guardrail_to_apply: "CustomGuardrail",
+ litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
+ user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
+ ) -> List[Any]:
+ """
+ Process A2A streaming output by applying guardrails to accumulated text.
+
+ responses_so_far can be a list of JSON-RPC 2.0 objects (dict or NDJSON str), e.g.:
+ - task with history, status-update, artifact-update (with result.artifact.parts),
+ - then status-update (final). Text is extracted from result.artifact.parts,
+ result.message.parts, result.parts, etc., concatenated in order, guardrailed once,
+ then the combined guardrailed text is written into the first chunk that had text
+ and all other text parts in other chunks are cleared (in-place).
+ """
+ from litellm.llms.a2a.common_utils import extract_text_from_a2a_response
+
+ # Parse each item; keep alignment with responses_so_far (None where unparseable)
+ parsed: List[Optional[Dict[str, Any]]] = [None] * len(responses_so_far)
+ for i, item in enumerate(responses_so_far):
+ if isinstance(item, dict):
+ obj = item
+ elif isinstance(item, str):
+ try:
+ obj = json.loads(item.strip())
+ except (json.JSONDecodeError, TypeError):
+ continue
+ else:
+ continue
+ if isinstance(obj.get("result"), dict):
+ parsed[i] = obj
+
+ valid_parsed = [(i, obj) for i, obj in enumerate(parsed) if obj is not None]
+ if not valid_parsed:
+ return responses_so_far
+
+ # Collect text from each chunk in order (by original index in responses_so_far)
+ text_parts: List[str] = []
+ chunk_indices_with_text: List[int] = [] # indices into valid_parsed
+ for idx, (orig_i, obj) in enumerate(valid_parsed):
+ t = extract_text_from_a2a_response(obj)
+ if t:
+ text_parts.append(t)
+ chunk_indices_with_text.append(orig_i)
+
+ combined_text = "".join(text_parts)
+ if not combined_text:
+ return responses_so_far
+
+ request_data: dict = {"responses_so_far": responses_so_far}
+ user_metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
+ if user_metadata:
+ request_data["litellm_metadata"] = user_metadata
+
+ inputs = GenericGuardrailAPIInputs(texts=[combined_text])
+ guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+ inputs=inputs,
+ request_data=request_data,
+ input_type="response",
+ logging_obj=litellm_logging_obj,
+ )
+ guardrailed_texts = guardrailed_inputs.get("texts", [])
+ if not guardrailed_texts:
+ return responses_so_far
+ guardrailed_text = guardrailed_texts[0]
+
+ # Find first chunk (by original index) that has text; put full guardrailed text there and clear rest
+ first_chunk_with_text: Optional[int] = (
+ chunk_indices_with_text[0] if chunk_indices_with_text else None
+ )
+
+ for orig_i, obj in valid_parsed:
+ result = obj.get("result", {})
+ if not isinstance(result, dict):
+ continue
+ texts_in_chunk: List[str] = []
+ mappings: List[Tuple[Tuple[str, ...], int]] = []
+ self._extract_texts_from_result(
+ result=result,
+ texts_to_check=texts_in_chunk,
+ task_mappings=mappings,
+ )
+ if not mappings:
+ continue
+ if orig_i == first_chunk_with_text:
+ # Put full guardrailed text in first text part; clear others
+ for task_idx, (path, part_idx) in enumerate(mappings):
+ text = guardrailed_text if task_idx == 0 else ""
+ self._apply_text_to_path(
+ result=result,
+ path=path,
+ part_idx=part_idx,
+ text=text,
+ )
+ else:
+ for path, part_idx in mappings:
+ self._apply_text_to_path(
+ result=result,
+ path=path,
+ part_idx=part_idx,
+ text="",
+ )
+
+ # Write back to responses_so_far where we had NDJSON strings
+ for i, item in enumerate(responses_so_far):
+ if isinstance(item, str) and parsed[i] is not None:
+ responses_so_far[i] = json.dumps(parsed[i]) + "\n"
+
+ return responses_so_far
+
def _extract_texts_from_result(
self,
result: Dict[str, Any],
diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py
index 9938cd7979b..c2cfff80685 100644
--- a/litellm/llms/anthropic/chat/transformation.py
+++ b/litellm/llms/anthropic/chat/transformation.py
@@ -208,29 +208,73 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
Filter out unsupported fields from JSON schema for Anthropic's output_format API.
Anthropic's output_format doesn't support certain JSON schema properties:
- - maxItems: Not supported for array types
- - minItems: Not supported for array types
+ - maxItems/minItems: Not supported for array types
+ - minimum/maximum: Not supported for numeric types
+ - minLength/maxLength: Not supported for string types
- This function recursively removes these unsupported fields while preserving
- all other valid schema properties.
+ This mirrors the transformation done by the Anthropic Python SDK.
+ See: https://platform.claude.com/docs/en/build-with-claude/structured-outputs#how-sdk-transformation-works
+
+ The SDK approach:
+ 1. Remove unsupported constraints from schema
+ 2. Add constraint info to description (e.g., "Must be at least 100")
+ 3. Validate responses against original schema
Args:
schema: The JSON schema dictionary to filter
Returns:
- A new dictionary with unsupported fields removed
+ A new dictionary with unsupported fields removed and descriptions updated
- Related issue: https://github.com/BerriAI/litellm/issues/19444
+ Related issues:
+ - https://github.com/BerriAI/litellm/issues/19444
"""
if not isinstance(schema, dict):
return schema
- unsupported_fields = {"maxItems", "minItems"}
+ # All numeric/string/array constraints not supported by Anthropic
+ unsupported_fields = {
+ "maxItems", "minItems", # array constraints
+ "minimum", "maximum", # numeric constraints
+ "exclusiveMinimum", "exclusiveMaximum", # numeric constraints
+ "minLength", "maxLength", # string constraints
+ }
+
+ # Build description additions from removed constraints
+ constraint_descriptions: list = []
+ constraint_labels = {
+ "minItems": "minimum number of items: {}",
+ "maxItems": "maximum number of items: {}",
+ "minimum": "minimum value: {}",
+ "maximum": "maximum value: {}",
+ "exclusiveMinimum": "exclusive minimum value: {}",
+ "exclusiveMaximum": "exclusive maximum value: {}",
+ "minLength": "minimum length: {}",
+ "maxLength": "maximum length: {}",
+ }
+ for field in unsupported_fields:
+ if field in schema:
+ constraint_descriptions.append(
+ constraint_labels[field].format(schema[field])
+ )
result: Dict[str, Any] = {}
+
+ # Update description with removed constraint info
+ if constraint_descriptions:
+ existing_desc = schema.get("description", "")
+ constraint_note = "Note: " + ", ".join(constraint_descriptions) + "."
+ if existing_desc:
+ result["description"] = existing_desc + " " + constraint_note
+ else:
+ result["description"] = constraint_note
+
for key, value in schema.items():
if key in unsupported_fields:
continue
+ if key == "description" and "description" in result:
+ # Already handled above
+ continue
if key == "properties" and isinstance(value, dict):
result[key] = {
diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py
index 44ce368fd49..78631d38005 100644
--- a/litellm/llms/azure/responses/transformation.py
+++ b/litellm/llms/azure/responses/transformation.py
@@ -1,5 +1,5 @@
-from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union
from copy import deepcopy
+from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union
import httpx
from openai.types.responses import ResponseReasoningItem
@@ -21,10 +21,25 @@ else:
class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
+
+ # Parameters not supported by Azure Responses API
+ AZURE_UNSUPPORTED_PARAMS = ["context_management"]
+
@property
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.AZURE
+ def get_supported_openai_params(self, model: str) -> list:
+ """
+ Azure Responses API does not support context_management (compaction).
+ """
+ base_supported_params = super().get_supported_openai_params(model)
+ return [
+ param
+ for param in base_supported_params
+ if param not in self.AZURE_UNSUPPORTED_PARAMS
+ ]
+
def validate_environment(
self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
) -> dict:
diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py
index 748a5f5fb40..51310e4fa85 100644
--- a/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py
+++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/llama3/transformation.py
@@ -1,12 +1,21 @@
import types
-from typing import Any, List, Optional
+from typing import Any, AsyncIterator, Iterator, List, Optional, Union
import httpx
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+from litellm.llms.openai.chat.gpt_transformation import (
+ OpenAIChatCompletionStreamingHandler,
+ OpenAIGPTConfig,
+)
from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionResponse
-from litellm.types.utils import ModelResponse, Usage
+from litellm.types.utils import (
+ Delta,
+ ModelResponse,
+ ModelResponseStream,
+ StreamingChoices,
+ Usage,
+)
from ...common_utils import VertexAIError
@@ -79,6 +88,18 @@ class VertexAILlama3Config(OpenAIGPTConfig):
drop_params=drop_params,
)
+ def get_model_response_iterator(
+ self,
+ streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
+ sync_stream: bool,
+ json_mode: Optional[bool] = False,
+ ) -> Any:
+ return VertexAILlama3StreamingHandler(
+ streaming_response=streaming_response,
+ sync_stream=sync_stream,
+ json_mode=json_mode,
+ )
+
def transform_response(
self,
model: str,
@@ -124,3 +145,80 @@ class VertexAILlama3Config(OpenAIGPTConfig):
)
return model_response
+
+
+class VertexAILlama3StreamingHandler(OpenAIChatCompletionStreamingHandler):
+ """
+ Vertex AI Llama models may not include role in streaming chunk deltas.
+ This handler ensures the first chunk always has role="assistant".
+
+ When Vertex AI returns a single chunk with both role and finish_reason (empty response),
+ this handler splits it into two chunks:
+ 1. First chunk: role="assistant", content="", finish_reason=None
+ 2. Second chunk: role=None, content=None, finish_reason="stop"
+
+ This matches OpenAI's streaming format where the first chunk has role and
+ the final chunk has finish_reason but no role.
+ """
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+ self.sent_role = False
+ self._pending_chunk: Optional[ModelResponseStream] = None
+
+ def chunk_parser(self, chunk: dict) -> ModelResponseStream:
+ result = super().chunk_parser(chunk)
+ if not self.sent_role and result.choices:
+ delta = result.choices[0].delta
+ finish_reason = result.choices[0].finish_reason
+
+ # If this is both the first chunk AND the final chunk (has finish_reason),
+ # we need to split it into two chunks to match OpenAI format
+ if finish_reason is not None:
+ # Create a pending final chunk with finish_reason but no role
+ self._pending_chunk = ModelResponseStream(
+ id=result.id,
+ object="chat.completion.chunk",
+ created=result.created,
+ model=result.model,
+ choices=[
+ StreamingChoices(
+ index=0,
+ delta=Delta(content=None, role=None),
+ finish_reason=finish_reason,
+ )
+ ],
+ )
+ # Modify current chunk to be the first chunk with role but no finish_reason
+ result.choices[0].finish_reason = None
+ delta.role = "assistant"
+ # Ensure content is empty string for first chunk, not None
+ if delta.content is None:
+ delta.content = ""
+ # Prevent downstream stream wrapper from dropping this chunk
+ # (it drops empty-content chunks unless special fields are present)
+ if delta.provider_specific_fields is None:
+ delta.provider_specific_fields = {}
+ elif delta.role is None:
+ delta.role = "assistant"
+ # If the first chunk has empty content, ensure it's still emitted
+ if (delta.content == "" or delta.content is None) and delta.provider_specific_fields is None:
+ delta.provider_specific_fields = {}
+ self.sent_role = True
+ return result
+
+ def __next__(self):
+ # First return any pending chunk from a previous split
+ if self._pending_chunk is not None:
+ chunk = self._pending_chunk
+ self._pending_chunk = None
+ return chunk
+ return super().__next__()
+
+ async def __anext__(self):
+ # First return any pending chunk from a previous split
+ if self._pending_chunk is not None:
+ chunk = self._pending_chunk
+ self._pending_chunk = None
+ return chunk
+ return await super().__anext__()
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index e6b7cf17297..95d8ba2ff60 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -6191,6 +6191,8 @@
"source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart",
"supports_function_calling": true,
"supports_reasoning": true,
+ "supports_system_messages": true,
+ "supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true
},
@@ -14835,7 +14837,9 @@
"supports_tool_choice": true,
"supports_url_context": true,
"supports_vision": true,
- "supports_web_search": true
+ "supports_web_search": true,
+ "tpm": 250000,
+ "rpm": 10
},
"gemini-2.5-computer-use-preview-10-2025": {
"input_cost_per_token": 1.25e-06,
@@ -16323,7 +16327,9 @@
"source": "https://ai.google.dev/pricing",
"supported_endpoints": [
"/v1/audio/speech"
- ]
+ ],
+ "tpm": 4000000,
+ "rpm": 10
},
"gemini/gemini-2.5-pro": {
"cache_read_input_token_cost": 1.25e-07,
@@ -16821,7 +16827,9 @@
"source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models",
"supports_function_calling": true,
"supports_tool_choice": true,
- "supports_vision": true
+ "supports_vision": true,
+ "tpm": 250000,
+ "rpm": 10
},
"gemini/gemini-gemma-2-9b-it": {
"input_cost_per_token": 3.5e-07,
@@ -16833,7 +16841,9 @@
"source": "https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#foundation_models",
"supports_function_calling": true,
"supports_tool_choice": true,
- "supports_vision": true
+ "supports_vision": true,
+ "tpm": 250000,
+ "rpm": 10
},
"gemini/gemini-pro": {
"input_cost_per_token": 3.5e-07,
@@ -23194,7 +23204,7 @@
"mode": "chat",
"output_cost_per_token": 6e-05,
"supports_function_calling": true,
- "supports_parallel_function_calling": true,
+ "supports_parallel_function_calling": false,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@@ -36495,7 +36505,9 @@
"text",
"image"
],
- "supports_vision": true
+ "supports_vision": true,
+ "tpm": 250000,
+ "rpm": 10
},
"gemini/gemini-2.0-flash-lite-001": {
"cache_read_input_token_cost": 1.875e-08,
@@ -36628,7 +36640,9 @@
"audio"
],
"supports_audio_input": true,
- "supports_audio_output": true
+ "supports_audio_output": true,
+ "tpm": 250000,
+ "rpm": 10
},
"gemini/gemini-2.5-flash-native-audio-preview-09-2025": {
"input_cost_per_audio_token": 1e-06,
@@ -36652,7 +36666,9 @@
"audio"
],
"supports_audio_input": true,
- "supports_audio_output": true
+ "supports_audio_output": true,
+ "tpm": 250000,
+ "rpm": 10
},
"gemini/gemini-2.5-flash-native-audio-preview-12-2025": {
"input_cost_per_audio_token": 1e-06,
@@ -36676,7 +36692,9 @@
"audio"
],
"supports_audio_input": true,
- "supports_audio_output": true
+ "supports_audio_output": true,
+ "tpm": 250000,
+ "rpm": 10
},
"gemini-2.5-flash-preview-tts": {
"input_cost_per_token": 3e-07,
diff --git a/litellm/policy_templates_backup.json b/litellm/policy_templates_backup.json
index 6541839611b..a0ffd6acd30 100644
--- a/litellm/policy_templates_backup.json
+++ b/litellm/policy_templates_backup.json
@@ -2,7 +2,7 @@
{
"id": "advanced-au-pii-protection",
"title": "Advanced PII Protection (Australia)",
- "description": "Comprehensive PII detection and masking for Australia. Protects Australian-specific identifiers, international employee data, financial information, credentials, protected class information, and industry-specific sensitive data.",
+ "description": "Protects Australian-specific identifiers, international employee data, financial information, credentials, protected class information, and industry-specific sensitive data.",
"icon": "ShieldCheckIcon",
"iconColor": "text-purple-500",
"iconBg": "bg-purple-50",
@@ -274,5 +274,405 @@
],
"guardrails_remove": []
}
+ },
+ {
+ "id": "nsfw-content-filter-australia",
+ "title": "NSFW Content Filter (Australia)",
+ "description": "Blocks profanity, sexual content, NSFW requests, self-harm content, and child safety violations using English and Australian-specific slang. Protects against inappropriate content including sexual solicitation, explicit content, Australian profanity, self-harm, and content involving minors.",
+ "icon": "ShieldExclamationIcon",
+ "iconColor": "text-red-500",
+ "iconBg": "bg-red-50",
+ "guardrails": [
+ "nsfw-content-filter-english",
+ "nsfw-content-filter-australian",
+ "nsfw-self-harm-filter",
+ "nsfw-child-safety-filter",
+ "nsfw-racial-bias-filter"
+ ],
+ "complexity": "Medium",
+ "guardrailDefinitions": [
+ {
+ "guardrail_name": "nsfw-content-filter-english",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harm_toxic_abuse",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Blocks profanity, sexual content, slurs, and NSFW terms in English"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-content-filter-australian",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harm_toxic_abuse_au",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Blocks Australian-specific slang and profanity (root, perv, bogan, wanker, etc.)"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-self-harm-filter",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harmful_self_harm",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Blocks content related to self-harm, suicide, and eating disorders"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-child-safety-filter",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harmful_child_safety",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Blocks inappropriate content involving minors using identifier + block word combinations"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-racial-bias-filter",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "bias_racial",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Blocks racial and ethnic discrimination, hate speech, and supremacist content"
+ }
+ }
+ ],
+ "templateData": {
+ "policy_name": "nsfw-content-filter-australia",
+ "description": "NSFW content filter for Australia. Blocks profanity, sexual content, inappropriate requests, self-harm content, child safety violations, and racial bias in English and Australian slang.",
+ "guardrails_add": [
+ "nsfw-content-filter-english",
+ "nsfw-content-filter-australian",
+ "nsfw-self-harm-filter",
+ "nsfw-child-safety-filter",
+ "nsfw-racial-bias-filter"
+ ],
+ "guardrails_remove": []
+ }
+ },
+ {
+ "id": "nsfw-content-filter-basic",
+ "title": "NSFW Content Filter (Basic)",
+ "description": "Basic NSFW content filtering for English only. Blocks profanity, sexual content, slurs, solicitation, explicit requests, self-harm content, and child safety violations. Suitable for most applications requiring content moderation.",
+ "icon": "ShieldExclamationIcon",
+ "iconColor": "text-orange-500",
+ "iconBg": "bg-orange-50",
+ "guardrails": [
+ "nsfw-content-filter-english-only",
+ "nsfw-self-harm-filter-basic",
+ "nsfw-child-safety-filter-basic",
+ "nsfw-racial-bias-filter-basic"
+ ],
+ "complexity": "Low",
+ "guardrailDefinitions": [
+ {
+ "guardrail_name": "nsfw-content-filter-english-only",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harm_toxic_abuse",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Blocks profanity, sexual content, slurs, and NSFW terms. Includes 485+ keywords covering explicit content, solicitation, sexual behavior, and exploitation."
+ }
+ },
+ {
+ "guardrail_name": "nsfw-self-harm-filter-basic",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harmful_self_harm",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Blocks content related to self-harm, suicide, and eating disorders"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-child-safety-filter-basic",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harmful_child_safety",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Blocks inappropriate content involving minors using identifier + block word combinations"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-racial-bias-filter-basic",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "bias_racial",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Blocks racial and ethnic discrimination, hate speech, and supremacist content"
+ }
+ }
+ ],
+ "templateData": {
+ "policy_name": "nsfw-content-filter-basic",
+ "description": "Basic NSFW content filter. Blocks profanity, sexual content, inappropriate requests, self-harm content, child safety violations, and racial bias in English.",
+ "guardrails_add": [
+ "nsfw-content-filter-english-only",
+ "nsfw-self-harm-filter-basic",
+ "nsfw-child-safety-filter-basic",
+ "nsfw-racial-bias-filter-basic"
+ ],
+ "guardrails_remove": []
+ }
+ },
+ {
+ "id": "nsfw-content-filter-all-regions",
+ "title": "NSFW Content Filter (All Regions)",
+ "description": "Comprehensive multi-language NSFW content filtering. Blocks profanity, sexual content, inappropriate requests, self-harm content, and child safety violations in English, Spanish, French, German, and Australian. Best for global applications.",
+ "icon": "ShieldExclamationIcon",
+ "iconColor": "text-purple-500",
+ "iconBg": "bg-purple-50",
+ "guardrails": [
+ "nsfw-filter-english",
+ "nsfw-filter-spanish",
+ "nsfw-filter-french",
+ "nsfw-filter-german",
+ "nsfw-filter-australian",
+ "nsfw-self-harm-filter-global",
+ "nsfw-child-safety-filter-global",
+ "nsfw-racial-bias-filter-global"
+ ],
+ "complexity": "High",
+ "guardrailDefinitions": [
+ {
+ "guardrail_name": "nsfw-filter-english",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harm_toxic_abuse",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "English profanity, sexual content, slurs, and NSFW terms (485+ keywords)"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-filter-spanish",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harm_toxic_abuse_es",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Spanish profanity and offensive terms (68 keywords)"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-filter-french",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harm_toxic_abuse_fr",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "French profanity and offensive terms (91 keywords)"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-filter-german",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harm_toxic_abuse_de",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "German profanity and offensive terms (65 keywords)"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-filter-australian",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harm_toxic_abuse_au",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Australian slang and profanity (32 keywords: root, perv, bogan, wanker, etc.)"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-self-harm-filter-global",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harmful_self_harm",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Blocks content related to self-harm, suicide, and eating disorders"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-child-safety-filter-global",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "harmful_child_safety",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Blocks inappropriate content involving minors using identifier + block word combinations"
+ }
+ },
+ {
+ "guardrail_name": "nsfw-racial-bias-filter-global",
+ "litellm_params": {
+ "guardrail": "litellm_content_filter",
+ "mode": "pre_call",
+ "categories": [
+ {
+ "category": "bias_racial",
+ "enabled": true,
+ "action": "BLOCK",
+ "severity_threshold": "medium"
+ }
+ ]
+ },
+ "guardrail_info": {
+ "description": "Blocks racial and ethnic discrimination, hate speech, and supremacist content"
+ }
+ }
+ ],
+ "templateData": {
+ "policy_name": "nsfw-content-filter-all-regions",
+ "description": "Comprehensive multi-language NSFW content filter. Blocks profanity, inappropriate content, self-harm, child safety violations, and racial bias in English, Spanish, French, German, and Australian. Total coverage: 741+ keywords across all languages plus self-harm, child safety, and racial bias protection.",
+ "guardrails_add": [
+ "nsfw-filter-english",
+ "nsfw-filter-spanish",
+ "nsfw-filter-french",
+ "nsfw-filter-german",
+ "nsfw-filter-australian",
+ "nsfw-self-harm-filter-global",
+ "nsfw-child-safety-filter-global",
+ "nsfw-racial-bias-filter-global"
+ ],
+ "guardrails_remove": []
+ }
}
]
diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html
deleted file mode 100644
index 788e9379f62..00000000000
--- a/litellm/proxy/_experimental/out/404.html
+++ /dev/null
@@ -1 +0,0 @@
-404: This page could not be found.LiteLLM Dashboard
404
This page could not be found.
\ No newline at end of file
diff --git a/litellm/proxy/_experimental/out/404/index.html b/litellm/proxy/_experimental/out/404/index.html
new file mode 100644
index 00000000000..47a4eda7e8b
--- /dev/null
+++ b/litellm/proxy/_experimental/out/404/index.html
@@ -0,0 +1 @@
+404: This page could not be found.LiteLLM Dashboard