scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

Merge remote-tracking branch 'origin/litellm_internal_staging' into codex/provider-url-destination-guard

Browse source
This commit is contained in:
user 2026-04-30 13:20:44 -07:00
commit e1f39674f3
31 changed files with 33823 additions and 292 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
.github/workflows/check-lazy-openapi-snapshot.yml vendored Normal file
View file

@ -0,0 +1,75 @@
name: Check Lazy OpenAPI Snapshot
on:
pull_request:
branches:
- main
- litellm_internal_staging
- "litellm_**"
permissions:
contents: read
checks: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
verify:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Cache uv dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Install dependencies
run: uv sync --frozen --all-groups --all-extras
- name: Regenerate snapshot to /tmp
id: regen
run: |
cp litellm/proxy/_lazy_openapi_snapshot.json /tmp/snapshot.committed.json
uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
mv litellm/proxy/_lazy_openapi_snapshot.json /tmp/snapshot.fresh.json
mv /tmp/snapshot.committed.json litellm/proxy/_lazy_openapi_snapshot.json
- name: Compare
id: diff
continue-on-error: true
run: |
diff -q /tmp/snapshot.fresh.json litellm/proxy/_lazy_openapi_snapshot.json
- name: Mark neutral if drift
if: steps.diff.outcome == 'failure'
uses: LouisBrunner/checks-action@6b626ffbad7cc56fd58627f774b9067e6118af23 # v2.0.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
name: lazy-openapi-snapshot
conclusion: neutral
output: |
{
"title": "Lazy openapi snapshot is stale",
"summary": "Run `python -m litellm.proxy._lazy_openapi_snapshot` and commit the regenerated `litellm/proxy/_lazy_openapi_snapshot.json`. Not blocking — the snapshot will regenerate at release if not committed."
}

17
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -23,6 +23,13 @@ def _raise_env_reference_error(param: str, *, source: str) -> None:
)
def validate_no_callback_env_reference(
param: str, value: object, *, source: str
) -> None:
if _is_env_reference(value):
_raise_env_reference_error(param, source=source)
# Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
_supported_callback_params = [
"langfuse_public_key",
@ -66,8 +73,9 @@ def initialize_standard_callback_dynamic_params(
for param in _supported_callback_params:
if param in kwargs:
_param_value = kwargs.get(param)
if _is_env_reference(_param_value):
_raise_env_reference_error(param, source="request body")
validate_no_callback_env_reference(
param, _param_value, source="request body"
)
standard_callback_dynamic_params[param] = _param_value # type: ignore
# 2. Fallback: check "metadata" or "litellm_params" -> "metadata"
@ -80,8 +88,9 @@ def initialize_standard_callback_dynamic_params(
for param in _supported_callback_params:
if param not in standard_callback_dynamic_params and param in metadata:
_param_value = metadata.get(param)
if _is_env_reference(_param_value):
_raise_env_reference_error(param, source="metadata")
validate_no_callback_env_reference(
param, _param_value, source="metadata"
)
standard_callback_dynamic_params[param] = _param_value # type: ignore
return standard_callback_dynamic_params

102
litellm/litellm_core_utils/redact_messages.py
View file

@ -60,6 +60,9 @@ def _redact_choice_content(choice):
def _redact_responses_api_output(output_items):
"""Helper to redact ResponsesAPIResponse output items."""
for output_item in output_items:
if hasattr(output_item, "text"):
output_item.text = "redacted-by-litellm"
if hasattr(output_item, "content") and isinstance(output_item.content, list):
for content_part in output_item.content:
if hasattr(content_part, "text"):
@ -75,6 +78,28 @@ def _redact_responses_api_output(output_items):
summary_item.text = "redacted-by-litellm"
def _redact_responses_api_output_dict(output_items, redacted_str: str):
"""Helper to redact ResponsesAPIResponse output items in dict form."""
for output_item in output_items:
if not isinstance(output_item, dict):
continue
if "text" in output_item:
output_item["text"] = redacted_str
if isinstance(output_item.get("content"), list):
for content_item in output_item["content"]:
if isinstance(content_item, dict) and "text" in content_item:
content_item["text"] = redacted_str
if output_item.get("type") == "reasoning" and isinstance(
output_item.get("summary"), list
):
for summary_item in output_item["summary"]:
if isinstance(summary_item, dict) and "text" in summary_item:
summary_item["text"] = redacted_str
def _redact_standard_logging_object(model_call_details: dict):
"""Redact messages and response inside standard_logging_object if present."""
standard_logging_object = model_call_details.get("standard_logging_object")
@ -93,28 +118,11 @@ def _redact_standard_logging_object(model_call_details: dict):
if isinstance(response, dict) and "output" in response:
# ResponsesAPIResponse format - redact content in output items
if isinstance(response.get("output"), list):
for output_item in response["output"]:
if isinstance(output_item, dict) and "content" in output_item:
if isinstance(output_item["content"], list):
for content_item in output_item["content"]:
if (
isinstance(content_item, dict)
and "text" in content_item
):
content_item["text"] = redacted_str
_redact_responses_api_output_dict(response["output"], redacted_str)
elif isinstance(response, dict) and "choices" in response:
# ModelResponse dict format - redact content in choices
if isinstance(response.get("choices"), list):
for choice in response["choices"]:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = redacted_str
if "audio" in choice["message"]:
choice["message"]["audio"] = None
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = redacted_str
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
_redact_model_response_dict_choices(response["choices"], redacted_str)
elif isinstance(response, str):
standard_logging_object["response"] = redacted_str
else:
@ -122,6 +130,29 @@ def _redact_standard_logging_object(model_call_details: dict):
standard_logging_object["response"] = {"text": redacted_str}
def _redact_model_response_dict_choices(choices, redacted_str: str):
for choice in choices:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = redacted_str
if "reasoning_content" in choice["message"]:
choice["message"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["message"]:
choice["message"]["thinking_blocks"] = None
if "audio" in choice["message"]:
choice["message"]["audio"] = None
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = redacted_str
if "reasoning_content" in choice["delta"]:
choice["delta"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["delta"]:
choice["delta"]["thinking_blocks"] = None
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
else:
_redact_choice_content(choice)
def perform_redaction(model_call_details: dict, result):
"""
Performs the actual redaction on the logging object and result.
@ -132,6 +163,7 @@ def perform_redaction(model_call_details: dict, result):
]
model_call_details["prompt"] = ""
model_call_details["input"] = ""
_redact_standard_logging_object(model_call_details)
# Redact streaming response
if (
@ -171,30 +203,14 @@ def perform_redaction(model_call_details: dict, result):
elif isinstance(_result, dict) and "choices" in _result:
# Handle dict representation of ModelResponse (e.g., from model_dump())
if _result.get("choices") is not None:
for choice in _result["choices"]:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = "redacted-by-litellm"
if "reasoning_content" in choice["message"]:
choice["message"][
"reasoning_content"
] = "redacted-by-litellm"
if "thinking_blocks" in choice["message"]:
choice["message"]["thinking_blocks"] = None
if "audio" in choice["message"]:
choice["message"]["audio"] = None
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = "redacted-by-litellm"
if "reasoning_content" in choice["delta"]:
choice["delta"][
"reasoning_content"
] = "redacted-by-litellm"
if "thinking_blocks" in choice["delta"]:
choice["delta"]["thinking_blocks"] = None
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
else:
_redact_choice_content(choice)
_redact_model_response_dict_choices(
_result["choices"], "redacted-by-litellm"
)
elif isinstance(_result, dict) and "output" in _result:
if isinstance(_result.get("output"), list):
_redact_responses_api_output_dict(
_result["output"], "redacted-by-litellm"
)
elif isinstance(_result, litellm.ResponsesAPIResponse):
if hasattr(_result, "output"):
_redact_responses_api_output(_result.output)

432
litellm/proxy/_lazy_features.py Normal file
View file

@ -0,0 +1,432 @@
"""
Lazy registration for optional feature routers. Each LAZY_FEATURES entry
imports its module only on the first request matching its path prefix,
saving ~700 MB at idle for deployments that don't use these features.
First hit pays the import cost (1-3 s for heavy modules); /openapi.json
omits each feature's routes until the feature is warmed.
"""
import asyncio
import importlib
import sys
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Callable, Dict, Tuple
from starlette.types import Receive, Scope, Send
from litellm._logging import verbose_proxy_logger
if TYPE_CHECKING:
from fastapi import APIRouter, FastAPI
def _include_router(attr_name: str = "router") -> Callable[["FastAPI", object], None]:
def _register(app: "FastAPI", module: object) -> None:
app.include_router(getattr(module, attr_name))
return _register
def _mount_app(
prefix: str, attr_name: str = "app"
) -> Callable[["FastAPI", object], None]:
def _register(app: "FastAPI", module: object) -> None:
app.mount(path=prefix, app=getattr(module, attr_name))
return _register
@dataclass(frozen=True)
class LazyFeature:
name: str
module_path: str
path_prefixes: Tuple[str, ...]
register_fn: Callable[["FastAPI", object], None] = field(
default_factory=lambda: _include_router("router")
)
# For routes whose path has a leading parameter (e.g. /{server}/authorize)
# — startswith can't match those, so the matcher also checks endswith.
path_suffixes: Tuple[str, ...] = ()
# Keep the stub injected even after load — for mounted ASGI sub-apps
# whose routes don't appear in the parent app's openapi spec.
persistent_swagger_stub: bool = False
LAZY_FEATURES: Tuple[LazyFeature, ...] = (
LazyFeature(
name="guardrails",
module_path="litellm.proxy.guardrails.guardrail_endpoints",
path_prefixes=(
"/guardrails",
"/v2/guardrails",
"/apply_guardrail",
"/policies/usage",
),
),
LazyFeature(
name="policies",
module_path="litellm.proxy.management_endpoints.policy_endpoints",
# Trailing slash to avoid matching /policies/... (policy_engine).
path_prefixes=("/policy/", "/utils/test_policies_and_guardrails"),
),
LazyFeature(
name="policy_engine",
module_path="litellm.proxy.policy_engine.policy_endpoints",
path_prefixes=("/policies",),
),
LazyFeature(
name="policy_resolve",
module_path="litellm.proxy.policy_engine.policy_resolve_endpoints",
path_prefixes=("/policies/resolve", "/policies/attachments/estimate-impact"),
),
LazyFeature(
name="agents",
module_path="litellm.proxy.agent_endpoints.endpoints",
path_prefixes=("/v1/agents", "/agents", "/agent/"),
),
LazyFeature(
name="a2a",
module_path="litellm.proxy.agent_endpoints.a2a_endpoints",
path_prefixes=("/a2a", "/v1/a2a"),
),
LazyFeature(
name="vector_stores",
module_path="litellm.proxy.vector_store_endpoints.endpoints",
path_prefixes=("/v1/vector_stores", "/vector_stores", "/v1/indexes"),
),
LazyFeature(
name="vector_store_management",
module_path="litellm.proxy.vector_store_endpoints.management_endpoints",
# Trailing slash to avoid matching /vector_stores/... (vector_stores).
path_prefixes=("/vector_store/", "/v1/vector_store/"),
),
LazyFeature(
name="vector_store_files",
# Routes appear under both /v1/vector_stores/{id}/files and the
# un-versioned form, so both prefixes must trigger the load.
module_path="litellm.proxy.vector_store_files_endpoints.endpoints",
path_prefixes=("/v1/vector_stores", "/vector_stores"),
),
LazyFeature(
name="tools",
module_path="litellm.proxy.management_endpoints.tool_management_endpoints",
path_prefixes=("/v1/tool", "/tool"),
),
LazyFeature(
name="search_tools",
module_path="litellm.proxy.search_endpoints.search_tool_management",
path_prefixes=("/search_tools",),
),
# mcp_management owns most /v1/mcp/* admin routes; mcp_app is the mounted
# streaming sub-app at /mcp.
LazyFeature(
name="mcp_management",
module_path="litellm.proxy.management_endpoints.mcp_management_endpoints",
path_prefixes=("/v1/mcp/",),
),
LazyFeature(
# Also serves /.well-known/oauth-* (OAuth metadata discovery).
# No /mcp/oauth prefix here: the mounted /mcp sub-app would
# shadow it, and there are no actual routes there anyway.
name="mcp_byok_oauth",
module_path="litellm.proxy._experimental.mcp_server.byok_oauth_endpoints",
path_prefixes=("/v1/mcp/oauth", "/.well-known/oauth-"),
),
LazyFeature(
# Serves OAuth dance endpoints (/authorize, /token, /callback,
# /register) plus several /.well-known/ discovery URLs at the proxy
# root — needed for MCP-over-OAuth flows even before /mcp is hit.
name="mcp_discoverable",
module_path="litellm.proxy._experimental.mcp_server.discoverable_endpoints",
path_prefixes=(
"/.well-known/oauth-",
"/.well-known/openid-configuration",
"/.well-known/jwks.json",
"/authorize",
"/token",
"/callback",
"/register",
),
# Catches the /{mcp_server_name}/authorize|token|register variants.
path_suffixes=("/authorize", "/token", "/register"),
),
LazyFeature(
name="mcp_rest",
module_path="litellm.proxy._experimental.mcp_server.rest_endpoints",
path_prefixes=("/mcp-rest",),
),
LazyFeature(
# Hardcoded /mcp matches BASE_MCP_ROUTE; importing the constant
# here would defeat lazy loading.
name="mcp_app",
module_path="litellm.proxy._experimental.mcp_server.server",
path_prefixes=("/mcp",),
register_fn=_mount_app("/mcp", attr_name="app"),
persistent_swagger_stub=True,
),
LazyFeature(
name="config_overrides",
module_path="litellm.proxy.management_endpoints.config_override_endpoints",
path_prefixes=("/config_overrides",),
),
LazyFeature(
name="realtime",
module_path="litellm.proxy.realtime_endpoints.endpoints",
path_prefixes=("/openai/v1/realtime", "/v1/realtime", "/realtime"),
),
LazyFeature(
name="anthropic_passthrough",
module_path="litellm.proxy.anthropic_endpoints.endpoints",
path_prefixes=("/v1/messages", "/anthropic", "/api/event_logging"),
),
LazyFeature(
name="anthropic_skills",
module_path="litellm.proxy.anthropic_endpoints.skills_endpoints",
path_prefixes=("/v1/skills", "/skills"),
),
LazyFeature(
name="langfuse_passthrough",
module_path="litellm.proxy.vertex_ai_endpoints.langfuse_endpoints",
path_prefixes=("/langfuse",),
),
LazyFeature(
name="evals",
module_path="litellm.proxy.openai_evals_endpoints.endpoints",
path_prefixes=("/v1/evals", "/evals"),
),
LazyFeature(
name="claude_code_marketplace",
module_path="litellm.proxy.anthropic_endpoints.claude_code_endpoints",
path_prefixes=("/claude-code",),
register_fn=_include_router("claude_code_marketplace_router"),
),
LazyFeature(
name="scim",
module_path="litellm.proxy.management_endpoints.scim.scim_v2",
path_prefixes=("/scim",),
register_fn=_include_router("scim_router"),
),
LazyFeature(
name="cloudzero",
module_path="litellm.proxy.spend_tracking.cloudzero_endpoints",
path_prefixes=("/cloudzero",),
),
LazyFeature(
name="vantage",
module_path="litellm.proxy.spend_tracking.vantage_endpoints",
path_prefixes=("/vantage",),
),
LazyFeature(
name="usage_ai",
module_path="litellm.proxy.management_endpoints.usage_endpoints",
path_prefixes=("/usage/ai",),
),
LazyFeature(
name="prompts",
module_path="litellm.proxy.prompts.prompt_endpoints",
path_prefixes=("/prompts", "/utils/dotprompt_json_converter"),
),
LazyFeature(
name="jwt_mappings",
module_path="litellm.proxy.management_endpoints.jwt_key_mapping_endpoints",
path_prefixes=("/jwt/key/mapping",),
),
LazyFeature(
name="compliance",
module_path="litellm.proxy.management_endpoints.compliance_endpoints",
path_prefixes=("/compliance",),
),
LazyFeature(
name="access_groups",
module_path="litellm.proxy.management_endpoints.access_group_endpoints",
path_prefixes=("/access_group", "/v1/access_group", "/v1/unified_access_group"),
),
)
class LazyFeatureMiddleware:
"""ASGI middleware that imports + registers a feature router on first
matching request. Idempotent; once loaded, subsequent requests skip."""
def __init__(
self,
app,
fastapi_app: "FastAPI",
features: Tuple[LazyFeature, ...] = LAZY_FEATURES,
):
self.app = app
self._fastapi_app = fastapi_app
self._features = features
# Loaded set / per-feature locks live on app.state so the warm endpoint
# and the middleware share them — preventing duplicate registrations
# when both paths fire for the same feature.
if not hasattr(fastapi_app.state, "lazy_loaded"):
fastapi_app.state.lazy_loaded = set()
fastapi_app.state.lazy_locks = {}
@property
def _loaded(self) -> set:
return self._fastapi_app.state.lazy_loaded
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
# Short-circuit once every feature has loaded.
if scope["type"] in ("http", "websocket") and len(self._loaded) < len(
self._features
):
path = scope.get("path", "")
for feat in self._features:
if feat.module_path in self._loaded:
continue
if any(path.startswith(p) for p in feat.path_prefixes) or any(
path.endswith(s) for s in feat.path_suffixes
):
await _force_load(self._fastapi_app, feat)
await self.app(scope, receive, send)
async def _force_load(app: "FastAPI", feat: LazyFeature) -> bool:
"""Import + register a lazy feature exactly once per (app, module).
Shared by the middleware and the /lazy/warm endpoint."""
if not hasattr(app.state, "lazy_loaded"):
app.state.lazy_loaded = set()
app.state.lazy_locks = {}
lock = app.state.lazy_locks.setdefault(feat.module_path, asyncio.Lock())
async with lock:
if feat.module_path in app.state.lazy_loaded:
return False
try:
# Import on a thread (heavy modules take 1-3 s). register_fn
# mutates app.router.routes, so it stays on the loop thread.
loop = asyncio.get_running_loop()
module = await loop.run_in_executor(
None, importlib.import_module, feat.module_path
)
feat.register_fn(app, module)
app.state.lazy_loaded.add(feat.module_path)
app.openapi_schema = None
verbose_proxy_logger.info(
"Lazy-loaded optional feature %r (module: %s)",
feat.name,
feat.module_path,
)
return True
except Exception as exc:
# Mark loaded anyway so we don't retry on every request.
app.state.lazy_loaded.add(feat.module_path)
verbose_proxy_logger.warning(
"Failed to lazy-load optional feature %r (module: %s): %s. "
"This feature's endpoints will return 404 until restart.",
feat.name,
feat.module_path,
exc,
)
return False
def attach_lazy_features(app: "FastAPI") -> None:
app.include_router(_make_warmup_router(app))
app.add_middleware(LazyFeatureMiddleware, fastapi_app=app)
def _make_warmup_router(app: "FastAPI") -> "APIRouter":
"""POST /lazy/warm/{name}: load a feature and return its partial openapi
so the Swagger plugin can merge in-place without a full /openapi.json refetch.
Requires auth — anyone who can hit the proxy can already trigger the same
imports by sending a real request to a feature's prefix, but gating this
debug endpoint avoids unauthenticated callers forcing the import chain."""
from fastapi import APIRouter, Depends, HTTPException
from fastapi.openapi.utils import get_openapi
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
router = APIRouter()
@router.post(
"/lazy/warm/{name}",
include_in_schema=False,
dependencies=[Depends(user_api_key_auth)],
)
async def warm(name: str):
feat = next((f for f in LAZY_FEATURES if f.name == name), None)
if feat is None:
raise HTTPException(404, f"unknown lazy feature: {name}")
if feat.persistent_swagger_stub:
return {"stub_path": None, "paths": {}, "components": {"schemas": {}}}
await _force_load(app, feat)
feat_routes = [
r
for r in app.routes
if any(getattr(r, "path", "").startswith(p) for p in feat.path_prefixes)
]
full = get_openapi(title=app.title, version=app.version, routes=feat_routes)
# Force all operations under one tag so they group under a single Swagger
# section — many lazy modules tag routes inconsistently.
for path_ops in full.get("paths", {}).values():
for op in path_ops.values():
if isinstance(op, dict):
op["tags"] = [feat.name]
return {
"stub_path": feat.path_prefixes[0],
"paths": full.get("paths", {}),
"components": {"schemas": full.get("components", {}).get("schemas", {})},
}
return router
def inject_lazy_stubs(schema: Dict) -> Dict:
"""Inject openapi entries for unloaded features. Uses the snapshot file
when available (full route info), otherwise falls back to a single
placeholder per feature. Any failure logs and returns the schema unchanged
so /openapi.json never 500s on a cosmetic injection bug."""
try:
from litellm.proxy._lazy_openapi_snapshot import load_snapshot
snapshot = load_snapshot()
paths = schema.setdefault("paths", {})
schemas = schema.setdefault("components", {}).setdefault("schemas", {})
for feat in LAZY_FEATURES:
if feat.module_path in sys.modules and not feat.persistent_swagger_stub:
continue
fragment = (snapshot or {}).get(feat.name)
if fragment:
for p, ops in fragment.get("paths", {}).items():
paths.setdefault(p, ops)
for name, sch in (
fragment.get("components", {}).get("schemas", {}).items()
):
schemas.setdefault(name, sch)
continue
prefix = feat.path_prefixes[0]
if prefix in paths:
continue
paths[prefix] = {
"get": {
"tags": [feat.name],
"summary": feat.name,
"responses": {"200": {"description": "OK"}},
}
}
except Exception as exc:
verbose_proxy_logger.warning("inject_lazy_stubs failed: %s", exc)
return schema
def lazy_tag_to_prefix() -> Dict[str, str]:
"""feature.name -> first prefix, used by the Swagger warmup JS plugin.
Returns empty when the snapshot is loaded — the plugin is unnecessary
because /openapi.json already has full route info."""
from litellm.proxy._lazy_openapi_snapshot import load_snapshot
if load_snapshot():
return {}
return {
feat.name: feat.path_prefixes[0]
for feat in LAZY_FEATURES
if not feat.persistent_swagger_stub
}

31651
litellm/proxy/_lazy_openapi_snapshot.json Normal file
View file

File diff suppressed because it is too large Load diff

70
litellm/proxy/_lazy_openapi_snapshot.py Normal file
View file

@ -0,0 +1,70 @@
"""
Per-feature OpenAPI snapshot for lazy-loaded routers.
The committed JSON is generated by `python -m litellm.proxy._lazy_openapi_snapshot`
and consumed at runtime so /openapi.json can show full route info for unloaded
features without importing them. CI verifies the file is current and surfaces
any drift as a neutral check.
"""
import json
import sys
from pathlib import Path
from typing import Dict, Optional
SNAPSHOT_FILE = Path(__file__).parent / "_lazy_openapi_snapshot.json"
def load_snapshot() -> Optional[Dict[str, Dict]]:
if not SNAPSHOT_FILE.exists():
return None
try:
with SNAPSHOT_FILE.open() as f:
return json.load(f)
except (json.JSONDecodeError, OSError):
return None
def generate_snapshot() -> Dict[str, Dict]:
import importlib
from fastapi.openapi.utils import get_openapi
from litellm.proxy._lazy_features import LAZY_FEATURES
from litellm.proxy.proxy_server import app
for feat in LAZY_FEATURES:
if feat.module_path in sys.modules:
continue
try:
module = importlib.import_module(feat.module_path)
feat.register_fn(app, module)
except Exception as exc:
sys.stderr.write(f"warning: skip {feat.name}: {exc}\n")
fragments: Dict[str, Dict] = {}
for feat in LAZY_FEATURES:
feat_routes = [
r
for r in app.routes
if any(getattr(r, "path", "").startswith(p) for p in feat.path_prefixes)
]
if not feat_routes:
continue
full = get_openapi(title=app.title, version=app.version, routes=feat_routes)
# Group all of a feature's routes under one tag.
for path_ops in full.get("paths", {}).values():
for op in path_ops.values():
if isinstance(op, dict):
op["tags"] = [feat.name]
fragments[feat.name] = {
"paths": full.get("paths", {}),
"components": {"schemas": full.get("components", {}).get("schemas", {})},
}
return fragments
if __name__ == "__main__":
fragments = generate_snapshot()
SNAPSHOT_FILE.write_text(json.dumps(fragments, indent=2, sort_keys=True) + "\n")
sys.stdout.write(f"wrote {len(fragments)} feature fragments to {SNAPSHOT_FILE}\n")

9
litellm/proxy/_types.py
View file

@ -17,6 +17,9 @@ from typing_extensions import Required, TypedDict
from litellm._uuid import uuid
from litellm.constants import MCP_STDIO_ALLOWED_COMMANDS
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
validate_no_callback_env_reference,
)
from litellm.types.integrations.slack_alerting import AlertType
from litellm.types.llms.openai import (
AllMessageValues,
@ -1869,8 +1872,10 @@ class AddTeamCallback(LiteLLMPydanticObjectBase):
raise ValueError(
f"Invalid callback variable: {key}. Must be one of {valid_keys}"
)
if not isinstance(value, str):
callback_vars[key] = str(value)
callback_vars[key] = str(value)
validate_no_callback_env_reference(
key, callback_vars[key], source="key/team callback metadata"
)
return values

17
litellm/proxy/common_utils/callback_utils.py
View file

@ -14,6 +14,8 @@ from litellm.types.utils import (
blue_color_code = "\033[94m"
reset_color_code = "\033[0m"
TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY = "_pillar_response_headers_trusted"
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
@ -417,10 +419,19 @@ def get_logging_caching_headers(request_data: Dict) -> Optional[Dict]:
if "semantic-similarity" in _metadata:
headers["x-litellm-semantic-similarity"] = str(_metadata["semantic-similarity"])
is_trusted_pillar_metadata = (
_metadata.get(TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY) is True
)
pillar_headers = _metadata.get("pillar_response_headers")
if isinstance(pillar_headers, dict):
headers.update(pillar_headers)
elif "pillar_flagged" in _metadata:
if is_trusted_pillar_metadata and isinstance(pillar_headers, dict):
headers.update(
{
key: str(value)
for key, value in pillar_headers.items()
if isinstance(key, str) and key.lower().startswith("x-pillar-")
}
)
elif is_trusted_pillar_metadata and "pillar_flagged" in _metadata:
headers["x-pillar-flagged"] = str(_metadata["pillar_flagged"]).lower()
return headers

10
litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
View file

@ -71,6 +71,7 @@ from litellm.types.utils import (
)
GUARDRAIL_NAME = "bedrock"
_BEDROCK_DYNAMIC_BODY_DENYLIST = frozenset({"content", "source"})
class GuardrailMessageFilterResult(NamedTuple):
@ -413,11 +414,18 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
)
api_key: Optional[str] = None
if request_data:
bedrock_request_data.update(
dynamic_request_body_params = (
self.get_guardrail_dynamic_request_body_params(
request_data=request_data
)
)
bedrock_request_data.update(
{
key: value
for key, value in dynamic_request_body_params.items()
if key not in _BEDROCK_DYNAMIC_BODY_DENYLIST
}
)
if request_data.get("api_key") is not None:
api_key = request_data["api_key"]

2
litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py
View file

@ -29,6 +29,7 @@ from litellm.llms.custom_httpx.http_handler import (
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.callback_utils import (
TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY,
add_guardrail_to_applied_guardrails_header,
get_metadata_variable_name_from_kwargs,
)
@ -144,6 +145,7 @@ def build_pillar_response_headers(metadata_store: Dict[str, Any]) -> Dict[str, s
if headers:
metadata_store["pillar_response_headers"] = headers
metadata_store[TRUSTED_PILLAR_RESPONSE_HEADERS_METADATA_KEY] = True
return headers

36
litellm/proxy/hooks/key_management_event_hooks.py
View file

@ -41,6 +41,7 @@ class KeyManagementEventHooks:
"""
from litellm.proxy.management_helpers.audit_logs import (
create_audit_log_for_update,
get_audit_log_changed_by,
)
from litellm.proxy.proxy_server import litellm_proxy_admin_name
@ -61,9 +62,11 @@ class KeyManagementEventHooks:
request_data=LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by=litellm_changed_by
or user_api_key_dict.user_id
or litellm_proxy_admin_name,
changed_by=get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
),
changed_by_api_key=user_api_key_dict.api_key,
table_name=LitellmTableNames.KEY_TABLE_NAME,
object_id=response.token_id or "",
@ -102,6 +105,7 @@ class KeyManagementEventHooks:
"""
from litellm.proxy.management_helpers.audit_logs import (
create_audit_log_for_update,
get_audit_log_changed_by,
)
from litellm.proxy.proxy_server import litellm_proxy_admin_name
@ -117,9 +121,11 @@ class KeyManagementEventHooks:
request_data=LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by=litellm_changed_by
or user_api_key_dict.user_id
or litellm_proxy_admin_name,
changed_by=get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
),
changed_by_api_key=user_api_key_dict.api_key,
table_name=LitellmTableNames.KEY_TABLE_NAME,
object_id=data.key,
@ -140,6 +146,7 @@ class KeyManagementEventHooks:
):
from litellm.proxy.management_helpers.audit_logs import (
create_audit_log_for_update,
get_audit_log_changed_by,
)
from litellm.proxy.proxy_server import litellm_proxy_admin_name
@ -189,9 +196,11 @@ class KeyManagementEventHooks:
request_data=LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by=litellm_changed_by
or user_api_key_dict.user_id
or litellm_proxy_admin_name,
changed_by=get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
),
changed_by_api_key=user_api_key_dict.token,
table_name=LitellmTableNames.KEY_TABLE_NAME,
object_id=existing_key_row.token,
@ -220,6 +229,7 @@ class KeyManagementEventHooks:
"""
from litellm.proxy.management_helpers.audit_logs import (
create_audit_log_for_update,
get_audit_log_changed_by,
)
from litellm.proxy.proxy_server import litellm_proxy_admin_name
@ -237,9 +247,11 @@ class KeyManagementEventHooks:
request_data=LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by=litellm_changed_by
or user_api_key_dict.user_id
or litellm_proxy_admin_name,
changed_by=get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
),
changed_by_api_key=user_api_key_dict.token,
table_name=LitellmTableNames.KEY_TABLE_NAME,
object_id=key.token,

12
litellm/proxy/hooks/user_management_event_hooks.py
View file

@ -192,13 +192,19 @@ class UserManagementEventHooks:
if not litellm.store_audit_logs:
return
from litellm.proxy.management_helpers.audit_logs import (
get_audit_log_changed_by,
)
await create_audit_log_for_update(
request_data=LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by=litellm_changed_by
or user_api_key_dict.user_id
or litellm_proxy_admin_name,
changed_by=get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
),
changed_by_api_key=user_api_key_dict.api_key,
table_name=LitellmTableNames.USER_TABLE_NAME,
object_id=user_id,

182
litellm/proxy/litellm_pre_call_utils.py
View file

@ -6,6 +6,7 @@ from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from fastapi import Request
from pydantic import ValidationError as PydanticValidationError
from starlette.datastructures import Headers
import litellm
@ -104,6 +105,112 @@ LITELLM_METADATA_ROUTES = (
"files",
)
_UNTRUSTED_ROOT_CONTROL_FIELDS = (
"proxy_server_request",
"standard_logging_object",
"secret_fields",
"mock_response",
"mock_tool_calls",
"disable_global_guardrails",
"disable_global_guardrail",
"opted_out_global_guardrails",
"applied_guardrails",
"applied_policies",
"policy_sources",
"pillar_response_headers",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",
)
_UNTRUSTED_METADATA_CONTROL_FIELDS = (
"disable_global_guardrails",
"disable_global_guardrail",
"opted_out_global_guardrails",
"pillar_response_headers",
"_pillar_response_headers_trusted",
"pillar_flagged",
"pillar_scanners",
"pillar_evidence",
"pillar_evidence_truncated",
"pillar_session_id_response",
"applied_guardrails",
"applied_policies",
"policy_sources",
"standard_logging_object",
"proxy_server_request",
"secret_fields",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",
)
_UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS = frozenset(
{
"litellm-disable-message-redaction",
}
)
_CLIENT_MOCK_CONTROL_FIELDS = frozenset({"mock_response", "mock_tool_calls"})
_ALLOW_CLIENT_MOCK_RESPONSE_METADATA_KEY = "allow_client_mock_response"
_ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY = (
"allow_client_message_redaction_opt_out"
)
def _strip_untrusted_request_header_controls(
headers: Any,
*,
allow_client_message_redaction_opt_out: bool = False,
) -> None:
if not isinstance(headers, dict):
return
for header_name in list(headers.keys()):
if (
isinstance(header_name, str)
and header_name.lower() in _UNTRUSTED_REQUEST_HEADER_CONTROL_FIELDS
):
if allow_client_message_redaction_opt_out:
continue
headers.pop(header_name, None)
def _is_false_like(value: Any) -> bool:
if isinstance(value, bool):
return value is False
if isinstance(value, str):
return value.strip().lower() in {"false", "0", "no", "off"}
return False
def _key_or_team_metadata_flag_is_true(
user_api_key_dict: UserAPIKeyAuth,
metadata_key: str,
) -> bool:
for admin_metadata in (user_api_key_dict.metadata, user_api_key_dict.team_metadata):
if (
isinstance(admin_metadata, dict)
and admin_metadata.get(metadata_key) is True
):
return True
return False
def _key_or_team_allows_client_mock_response(
user_api_key_dict: UserAPIKeyAuth,
) -> bool:
return _key_or_team_metadata_flag_is_true(
user_api_key_dict=user_api_key_dict,
metadata_key=_ALLOW_CLIENT_MOCK_RESPONSE_METADATA_KEY,
)
def _key_or_team_allows_client_message_redaction_opt_out(
user_api_key_dict: UserAPIKeyAuth,
) -> bool:
return _key_or_team_metadata_flag_is_true(
user_api_key_dict=user_api_key_dict,
metadata_key=_ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY,
)
def _get_metadata_variable_name(request: Request) -> str:
"""
@ -228,13 +335,25 @@ def convert_key_logging_metadata_to_callback(
for var, value in data.callback_vars.items():
if team_callback_settings_obj.callback_vars is None:
team_callback_settings_obj.callback_vars = {}
team_callback_settings_obj.callback_vars[var] = str(
litellm.utils.get_secret(value, default_value=value) or value
)
team_callback_settings_obj.callback_vars[var] = str(value)
return team_callback_settings_obj
def _get_validated_callback_metadata(
item: dict, *, source: str
) -> Optional[AddTeamCallback]:
try:
return AddTeamCallback(**item)
except (PydanticValidationError, ValueError) as e:
verbose_proxy_logger.warning(
"Ignoring invalid %s callback metadata: %s",
source,
_sanitize_for_log(str(e)),
)
return None
class KeyAndTeamLoggingSettings:
"""
Helper class to get the dynamic logging settings for the key and team
@ -274,8 +393,11 @@ def _get_dynamic_logging_metadata(
#########################################################################################
if key_dynamic_logging_settings is not None:
for item in key_dynamic_logging_settings:
callback = _get_validated_callback_metadata(item=item, source="key-level")
if callback is None:
continue
callback_settings_obj = convert_key_logging_metadata_to_callback(
data=AddTeamCallback(**item),
data=callback,
team_callback_settings_obj=callback_settings_obj,
)
#########################################################################################
@ -283,8 +405,11 @@ def _get_dynamic_logging_metadata(
#########################################################################################
elif team_dynamic_logging_settings is not None:
for item in team_dynamic_logging_settings:
callback = _get_validated_callback_metadata(item=item, source="team-level")
if callback is None:
continue
callback_settings_obj = convert_key_logging_metadata_to_callback(
data=AddTeamCallback(**item),
data=callback,
team_callback_settings_obj=callback_settings_obj,
)
#########################################################################################
@ -904,6 +1029,14 @@ class LiteLLMProxyRequestSetup:
callback_vars_dict.pop("team_id", None)
callback_vars_dict.pop("success_callback", None)
callback_vars_dict.pop("failure_callback", None)
callback_vars_dict = {
key: (
litellm.utils.get_secret(value, default_value=value) or value
if isinstance(value, str)
else value
)
for key, value in callback_vars_dict.items()
}
return TeamCallbackMetadata(
success_callback=team_config.get("success_callback", None),
@ -962,11 +1095,15 @@ async def add_litellm_data_to_request( # noqa: PLR0915
# Strip internal-only keys from user input before the proxy sets its own.
# These keys are injected by the proxy itself below — user-supplied values
# must not be trusted.
for _internal_key in (
"proxy_server_request",
"standard_logging_object",
"secret_fields",
):
_allow_client_mock_response = _key_or_team_allows_client_mock_response(
user_api_key_dict
)
_allow_client_message_redaction_opt_out = (
_key_or_team_allows_client_message_redaction_opt_out(user_api_key_dict)
)
for _internal_key in _UNTRUSTED_ROOT_CONTROL_FIELDS:
if _allow_client_mock_response and _internal_key in _CLIENT_MOCK_CONTROL_FIELDS:
continue
data.pop(_internal_key, None)
# Strip spoofable auth metadata from user-supplied metadata dict
_user_metadata = data.get("metadata")
@ -1007,6 +1144,17 @@ async def add_litellm_data_to_request( # noqa: PLR0915
forward_llm_provider_auth_headers=forward_llm_auth,
authenticated_with_header=authenticated_with_header,
)
_strip_untrusted_request_header_controls(
_headers,
allow_client_message_redaction_opt_out=_allow_client_message_redaction_opt_out,
)
if (
not _allow_client_message_redaction_opt_out
and litellm.turn_off_message_logging is True
and "turn_off_message_logging" in data
and _is_false_like(data["turn_off_message_logging"])
):
data.pop("turn_off_message_logging", None)
verbose_proxy_logger.debug(f"Request Headers: {_headers}")
verbose_proxy_logger.debug(f"Raw Headers: {_raw_headers}")
@ -1144,8 +1292,18 @@ async def add_litellm_data_to_request( # noqa: PLR0915
for _meta_key in ("metadata", "litellm_metadata"):
_user_meta = data.get(_meta_key)
if isinstance(_user_meta, dict):
_user_meta.pop("_pipeline_managed_guardrails", None)
for _k in [k for k in _user_meta if k.startswith("user_api_key_")]:
_strip_untrusted_request_header_controls(
_user_meta.get("headers"),
allow_client_message_redaction_opt_out=(
_allow_client_message_redaction_opt_out
),
)
for _k in [
k
for k in _user_meta
if k.startswith("user_api_key_")
or k in _UNTRUSTED_METADATA_CONTROL_FIELDS
]:
_user_meta.pop(_k, None)
# Strip caller-supplied routing/budget tags unless the admin has opted

11
litellm/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -2069,6 +2069,9 @@ async def delete_user(
litellm_proxy_admin_name,
prisma_client,
)
from litellm.proxy.management_helpers.audit_logs import (
get_audit_log_changed_by,
)
if prisma_client is None:
raise HTTPException(status_code=500, detail={"error": "No db connected"})
@ -2162,9 +2165,11 @@ async def delete_user(
request_data=LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by=litellm_changed_by
or user_api_key_dict.user_id
or litellm_proxy_admin_name,
changed_by=get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
),
changed_by_api_key=user_api_key_dict.api_key,
table_name=LitellmTableNames.USER_TABLE_NAME,
object_id=user_id,

24
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -5254,6 +5254,9 @@ async def block_key(
proxy_logging_obj,
user_api_key_cache,
)
from litellm.proxy.management_helpers.audit_logs import (
get_audit_log_changed_by,
)
if prisma_client is None:
raise Exception("{}".format(CommonProxyErrors.db_not_connected_error.value))
@ -5297,9 +5300,11 @@ async def block_key(
request_data=LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by=litellm_changed_by
or user_api_key_dict.user_id
or litellm_proxy_admin_name,
changed_by=get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
),
changed_by_api_key=user_api_key_dict.api_key,
table_name=LitellmTableNames.KEY_TABLE_NAME,
object_id=hashed_token,
@ -5363,6 +5368,9 @@ async def unblock_key(
proxy_logging_obj,
user_api_key_cache,
)
from litellm.proxy.management_helpers.audit_logs import (
get_audit_log_changed_by,
)
if prisma_client is None:
raise Exception("{}".format(CommonProxyErrors.db_not_connected_error.value))
@ -5406,9 +5414,11 @@ async def unblock_key(
request_data=LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by=litellm_changed_by
or user_api_key_dict.user_id
or litellm_proxy_admin_name,
changed_by=get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
),
changed_by_api_key=user_api_key_dict.api_key,
table_name=LitellmTableNames.KEY_TABLE_NAME,
object_id=hashed_token,
@ -5589,7 +5599,6 @@ async def test_key_logging(
"content": "Hello, this is a test from litellm /key/health. No LLM API call was made for this",
}
],
"mock_response": "test response",
}
data = await add_litellm_data_to_request(
data=data,
@ -5598,6 +5607,7 @@ async def test_key_logging(
general_settings=general_settings,
request=request,
)
data["mock_response"] = "test response"
await litellm.acompletion(
**data
) # make mock completion call to trigger key based callbacks

15
litellm/proxy/management_endpoints/mcp_management_endpoints.py
View file

@ -56,6 +56,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
)
from litellm.proxy.management_helpers.audit_logs import get_audit_log_changed_by
router = APIRouter(prefix="/v1/mcp", tags=["mcp"])
@ -2230,7 +2231,12 @@ if MCP_AVAILABLE:
detail={"error": "Only proxy admins can create MCP toolsets."},
)
touched_by = (
litellm_changed_by or user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME
get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=LITELLM_PROXY_ADMIN_NAME,
)
or LITELLM_PROXY_ADMIN_NAME
)
try:
result = await create_mcp_toolset(prisma_client, payload, touched_by)
@ -2321,7 +2327,12 @@ if MCP_AVAILABLE:
detail={"error": "Only proxy admins can update MCP toolsets."},
)
touched_by = (
litellm_changed_by or user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME
get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=LITELLM_PROXY_ADMIN_NAME,
)
or LITELLM_PROXY_ADMIN_NAME
)
try:
result = await update_mcp_toolset(prisma_client, payload, touched_by)

35
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -906,6 +906,9 @@ async def new_team( # noqa: PLR0915
prisma_client,
user_api_key_cache,
)
from litellm.proxy.management_helpers.audit_logs import (
get_audit_log_changed_by,
)
if prisma_client is None:
raise HTTPException(status_code=500, detail={"error": "No db connected"})
@ -1174,9 +1177,11 @@ async def new_team( # noqa: PLR0915
request_data=LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by=litellm_changed_by
or user_api_key_dict.user_id
or litellm_proxy_admin_name,
changed_by=get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
),
changed_by_api_key=user_api_key_dict.api_key,
table_name=LitellmTableNames.TEAM_TABLE_NAME,
object_id=data.team_id,
@ -1214,7 +1219,10 @@ async def _create_team_update_audit_log(
user_api_key_dict: User API key authentication details
litellm_proxy_admin_name: Name of the proxy admin
"""
from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update
from litellm.proxy.management_helpers.audit_logs import (
create_audit_log_for_update,
get_audit_log_changed_by,
)
_before_value = existing_team_row.json(exclude_none=True)
_before_value = json.dumps(_before_value, default=str)
@ -1225,9 +1233,11 @@ async def _create_team_update_audit_log(
request_data=LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by=litellm_changed_by
or user_api_key_dict.user_id
or litellm_proxy_admin_name,
changed_by=get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
),
changed_by_api_key=user_api_key_dict.api_key,
table_name=LitellmTableNames.TEAM_TABLE_NAME,
object_id=team_id,
@ -3037,6 +3047,9 @@ async def delete_team(
litellm_proxy_admin_name,
prisma_client,
)
from litellm.proxy.management_helpers.audit_logs import (
get_audit_log_changed_by,
)
if prisma_client is None:
raise HTTPException(status_code=500, detail={"error": "No db connected"})
@ -3096,9 +3109,11 @@ async def delete_team(
request_data=LiteLLM_AuditLogs(
id=str(uuid.uuid4()),
updated_at=datetime.now(timezone.utc),
changed_by=litellm_changed_by
or user_api_key_dict.user_id
or litellm_proxy_admin_name,
changed_by=get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
),
changed_by_api_key=user_api_key_dict.api_key,
table_name=LitellmTableNames.TEAM_TABLE_NAME,
object_id=team_id,

28
litellm/proxy/management_helpers/audit_logs.py
View file

@ -21,6 +21,28 @@ from litellm.proxy._types import (
from litellm.types.utils import StandardAuditLogPayload
_audit_log_callback_cache: Dict[str, CustomLogger] = {}
ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY = "allow_litellm_changed_by_header"
def _allows_litellm_changed_by_header(user_api_key_dict: UserAPIKeyAuth) -> bool:
for admin_metadata in (user_api_key_dict.metadata, user_api_key_dict.team_metadata):
if (
isinstance(admin_metadata, dict)
and admin_metadata.get(ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY) is True
):
return True
return False
def get_audit_log_changed_by(
*,
litellm_changed_by: Optional[str],
user_api_key_dict: UserAPIKeyAuth,
litellm_proxy_admin_name: Optional[str],
) -> Optional[str]:
if litellm_changed_by and _allows_litellm_changed_by_header(user_api_key_dict):
return litellm_changed_by
return user_api_key_dict.user_id or litellm_proxy_admin_name
def _resolve_audit_log_callback(name: str) -> Optional[CustomLogger]:
@ -143,8 +165,10 @@ async def create_object_audit_log(
if _store_audit_logs is not True:
return
_changed_by = (
litellm_changed_by or user_api_key_dict.user_id or litellm_proxy_admin_name
_changed_by = get_audit_log_changed_by(
litellm_changed_by=litellm_changed_by,
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name=litellm_proxy_admin_name,
)
await create_audit_log_for_update(

202
litellm/proxy/proxy_server.py
View file

@ -236,37 +236,11 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import (
router as mcp_byok_oauth_router,
)
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
router as mcp_discoverable_endpoints_router,
)
from litellm.proxy._experimental.mcp_server.rest_endpoints import (
router as mcp_rest_endpoints_router,
)
from litellm.proxy._experimental.mcp_server.server import app as mcp_app
from litellm.proxy._experimental.mcp_server.tool_registry import (
global_mcp_tool_registry,
)
from litellm.proxy._types import *
from litellm.proxy.agent_endpoints.a2a_endpoints import router as a2a_router
from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
from litellm.proxy.agent_endpoints.endpoints import router as agent_endpoints_router
from litellm.proxy.agent_endpoints.model_list_helpers import (
append_agents_to_model_group,
append_agents_to_model_info,
)
from litellm.proxy._lazy_features import attach_lazy_features
from litellm.proxy.analytics_endpoints.analytics_endpoints import (
router as analytics_router,
)
from litellm.proxy.anthropic_endpoints.claude_code_endpoints import (
claude_code_marketplace_router,
)
from litellm.proxy.anthropic_endpoints.endpoints import router as anthropic_router
from litellm.proxy.anthropic_endpoints.skills_endpoints import (
router as anthropic_skills_router,
)
from litellm.proxy.auth.auth_checks import (
ExperimentalUIJWTToken,
get_team_object,
@ -329,7 +303,6 @@ from litellm.proxy.discovery_endpoints import ui_discovery_endpoints_router
from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router
from litellm.proxy.fine_tuning_endpoints.endpoints import set_fine_tuning_config
from litellm.proxy.google_endpoints.endpoints import router as google_router
from litellm.proxy.guardrails.guardrail_endpoints import router as guardrails_router
from litellm.proxy.guardrails.init_guardrails import (
init_guardrails_v2,
initialize_guardrails,
@ -345,9 +318,6 @@ from litellm.proxy.hooks.prompt_injection_detection import (
from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger
from litellm.proxy.image_endpoints.endpoints import router as image_router
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
from litellm.proxy.management_endpoints.access_group_endpoints import (
router as access_group_router,
)
from litellm.proxy.management_endpoints.budget_management_endpoints import (
router as budget_management_router,
)
@ -361,12 +331,6 @@ from litellm.proxy.management_endpoints.common_utils import (
_user_has_admin_privileges,
admin_can_invite_user,
)
from litellm.proxy.management_endpoints.compliance_endpoints import (
router as compliance_router,
)
from litellm.proxy.management_endpoints.config_override_endpoints import (
router as config_override_router,
)
from litellm.proxy.management_endpoints.cost_tracking_settings import (
router as cost_tracking_settings_router,
)
@ -380,9 +344,6 @@ from litellm.proxy.management_endpoints.internal_user_endpoints import (
router as internal_user_router,
)
from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
from litellm.proxy.management_endpoints.jwt_key_mapping_endpoints import (
router as jwt_key_mapping_router,
)
from litellm.proxy.management_endpoints.key_management_endpoints import (
delete_verification_tokens,
duration_in_seconds,
@ -391,9 +352,6 @@ from litellm.proxy.management_endpoints.key_management_endpoints import (
from litellm.proxy.management_endpoints.key_management_endpoints import (
router as key_management_router,
)
from litellm.proxy.management_endpoints.mcp_management_endpoints import (
router as mcp_management_router,
)
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
router as model_access_group_management_router,
)
@ -408,11 +366,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import (
from litellm.proxy.management_endpoints.organization_endpoints import (
router as organization_router,
)
from litellm.proxy.management_endpoints.policy_endpoints import router as policy_router
from litellm.proxy.management_endpoints.router_settings_endpoints import (
router as router_settings_router,
)
from litellm.proxy.management_endpoints.scim.scim_v2 import scim_router
from litellm.proxy.management_endpoints.tag_management_endpoints import (
router as tag_management_router,
)
@ -424,9 +380,6 @@ from litellm.proxy.management_endpoints.team_endpoints import (
update_team,
validate_membership,
)
from litellm.proxy.management_endpoints.tool_management_endpoints import (
router as tool_management_router,
)
from litellm.proxy.management_endpoints.workflow_management_endpoints import (
router as workflow_management_router,
)
@ -435,7 +388,6 @@ from litellm.proxy.management_endpoints.ui_sso import (
get_disabled_non_admin_personal_key_creation,
)
from litellm.proxy.management_endpoints.ui_sso import router as ui_sso_router
from litellm.proxy.management_endpoints.usage_endpoints import router as usage_ai_router
from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import (
router as user_agent_analytics_router,
)
@ -445,7 +397,6 @@ from litellm.proxy.middleware.in_flight_requests_middleware import (
)
from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware
from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router
from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_router
from litellm.proxy.openai_files_endpoints.files_endpoints import (
router as openai_files_router,
)
@ -465,27 +416,16 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
router as pass_through_router,
)
from litellm.proxy.policy_engine.policy_endpoints import router as policy_crud_router
from litellm.proxy.policy_engine.policy_resolve_endpoints import (
router as policy_resolve_router,
)
from litellm.proxy.prompts.prompt_endpoints import router as prompts_router
from litellm.proxy.public_endpoints import router as public_endpoints_router
from litellm.proxy.rag_endpoints.endpoints import router as rag_router
from litellm.proxy.realtime_endpoints.endpoints import router as webrtc_router
from litellm.proxy.rerank_endpoints.endpoints import router as rerank_router
from litellm.proxy.response_api_endpoints.endpoints import router as response_router
from litellm.proxy.route_llm_request import route_request
from litellm.proxy.search_endpoints.endpoints import router as search_router
from litellm.proxy.search_endpoints.search_tool_management import (
router as search_tool_management_router,
)
from litellm.proxy.spend_tracking.cloudzero_endpoints import router as cloudzero_router
from litellm.proxy.spend_tracking.spend_management_endpoints import (
router as spend_management_router,
)
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.proxy.spend_tracking.vantage_endpoints import router as vantage_router
from litellm.proxy.types_utils.utils import get_instance_fn
from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import (
router as ui_crud_endpoints_router,
@ -515,16 +455,6 @@ from litellm.proxy.utils import (
prefetch_config_params,
update_spend,
)
from litellm.proxy.vector_store_endpoints.endpoints import router as vector_store_router
from litellm.proxy.vector_store_endpoints.management_endpoints import (
router as vector_store_management_router,
)
from litellm.proxy.vector_store_files_endpoints.endpoints import (
router as vector_store_files_router,
)
from litellm.proxy.vertex_ai_endpoints.langfuse_endpoints import (
router as langfuse_router,
)
from litellm.proxy.video_endpoints.endpoints import router as video_router
from litellm.router import (
AssistantsTypedDict,
@ -1104,6 +1034,11 @@ def get_openapi_schema():
openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema)
# Stub unloaded lazy features so they appear as Swagger sections.
from litellm.proxy._lazy_features import inject_lazy_stubs
openapi_schema = inject_lazy_stubs(openapi_schema)
# Fix Swagger UI execute path error when server_root_path is set
if server_root_path:
openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}]
@ -1130,6 +1065,11 @@ def custom_openapi():
openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema)
# Stub unloaded lazy features so they appear as Swagger sections.
from litellm.proxy._lazy_features import inject_lazy_stubs
openapi_schema = inject_lazy_stubs(openapi_schema)
# Fix Swagger UI execute path error when server_root_path is set
if server_root_path:
openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}]
@ -1557,14 +1497,78 @@ def mount_swagger_ui():
app.mount("/swagger", StaticFiles(directory=swagger_directory), name="swagger")
# On dropdown expand: one-time fetch to the prefix (triggers lazy load),
# then spec re-download so real routes replace the stub. Raw JS (no
# <script> tag) since it's injected inside the existing inline script.
from fastapi.responses import HTMLResponse
from litellm.proxy._lazy_features import lazy_tag_to_prefix
_lazy_plugin_js = (
"const TAG_TO_PREFIX = " + json.dumps(lazy_tag_to_prefix()) + ";"
"const warmedTags = new Set();"
"const LAZY_TAGS = new Set(Object.keys(TAG_TO_PREFIX));"
"const hideStubRows = () => {"
"document.querySelectorAll('.opblock').forEach(op => {"
"const d = op.querySelector('.opblock-summary-description');"
"if (d && LAZY_TAGS.has(d.textContent.trim())) op.style.display = 'none';"
"});};"
"const annotateLazyHeaders = () => {"
"document.querySelectorAll('.opblock-tag').forEach(tagEl => {"
"const m = (tagEl.id || '').match(/^operations-tag-(.+)$/);"
"if (!m || !LAZY_TAGS.has(m[1])) return;"
"const existing = tagEl.querySelector('.lazy-load-hint');"
"if (warmedTags.has(m[1])) { if (existing) existing.remove(); return; }"
"if (existing) return;"
"const hint = document.createElement('small');"
"hint.className = 'lazy-load-hint';"
"hint.textContent = ' (expand to load routes)';"
"hint.style.opacity = '0.6';"
"hint.style.marginLeft = '6px';"
"const target = tagEl.querySelector('a span') || tagEl.querySelector('span') || tagEl;"
"target.appendChild(hint);"
"});};"
"setInterval(() => { hideStubRows(); annotateLazyHeaders(); }, 200);"
"const LazyLoadPlugin = () => ({"
"afterLoad:function(system){setTimeout(()=>{"
"for(const tag of LAZY_TAGS)system.layoutActions.show(['operations-tag',tag],false);"
"},200);},"
"statePlugins:{layout:{wrapActions:{show:(ori,sys)=>(...args)=>{"
"const thing=args[0];const shown=args[1];let tag=null;"
"if(Array.isArray(thing)){for(const t of thing)if(TAG_TO_PREFIX[t])tag=t;}"
"if(shown!==false&&tag&&!warmedTags.has(tag)){warmedTags.add(tag);"
"fetch('/lazy/warm/'+tag,{method:'POST',credentials:'include'}).then(r=>r.json()).then(d=>{"
"if(!d.paths||Object.keys(d.paths).length===0)return;"
"const cur=sys.specSelectors.specJson().toJS();"
"const merged={};let inserted=false;"
"for(const k in (cur.paths||{})){"
"if(k===d.stub_path){for(const nk in d.paths)merged[nk]=d.paths[nk];inserted=true;}"
"else{merged[k]=cur.paths[k];}}"
"if(!inserted)Object.assign(merged,d.paths);"
"cur.paths=merged;"
"cur.components=cur.components||{};"
"cur.components.schemas=Object.assign(cur.components.schemas||{},(d.components||{}).schemas||{});"
"sys.specActions.updateSpec(JSON.stringify(cur));"
"}).catch(()=>{});}"
"return ori(...args);}}}}});"
)
def swagger_monkey_patch(*args, **kwargs):
return get_swagger_ui_html(
response = get_swagger_ui_html(
*args,
**kwargs,
swagger_js_url=f"{custom_root_path_swagger_path}/swagger-ui-bundle.js",
swagger_css_url=f"{custom_root_path_swagger_path}/swagger-ui.css",
swagger_favicon_url=f"{custom_root_path_swagger_path}/favicon.png",
)
body = response.body.decode("utf-8")
body = body.replace(
"const ui = SwaggerUIBundle({",
_lazy_plugin_js
+ 'const ui = SwaggerUIBundle({plugins:[LazyLoadPlugin],tagsSorter:"alpha",',
1,
)
return HTMLResponse(content=body)
applications.get_swagger_ui_html = swagger_monkey_patch
@ -3865,11 +3869,19 @@ class ProxyConfig:
## MCP TOOLS
mcp_tools_config = config.get("mcp_tools", None)
if mcp_tools_config:
from litellm.proxy._experimental.mcp_server.tool_registry import (
global_mcp_tool_registry,
)
global_mcp_tool_registry.load_tools_from_config(mcp_tools_config)
## AGENTS
agent_config = config.get("agent_list", None)
if agent_config:
from litellm.proxy.agent_endpoints.agent_registry import (
global_agent_registry,
)
global_agent_registry.load_agents_from_config(agent_config) # type: ignore
mcp_servers_config = config.get("mcp_servers", None)
@ -10587,6 +10599,10 @@ async def model_info_v2(
verbose_proxy_logger.debug("all_models: %s", all_models)
# Append A2A agents to models list
from litellm.proxy.agent_endpoints.model_list_helpers import (
append_agents_to_model_info,
)
all_models = await append_agents_to_model_info(
models=all_models,
user_api_key_dict=user_api_key_dict,
@ -11436,6 +11452,10 @@ async def model_group_info(
)
# Append A2A agents to model groups
from litellm.proxy.agent_endpoints.model_list_helpers import (
append_agents_to_model_group,
)
model_groups = await append_agents_to_model_group(
model_groups=model_groups,
user_api_key_dict=user_api_key_dict,
@ -14398,66 +14418,41 @@ app.include_router(container_router)
app.include_router(search_router)
app.include_router(image_router)
app.include_router(fine_tuning_router)
app.include_router(vector_store_router)
app.include_router(vector_store_management_router)
app.include_router(vector_store_files_router)
app.include_router(credential_router)
app.include_router(llm_passthrough_router)
app.include_router(webrtc_router)
app.include_router(mcp_management_router)
app.include_router(mcp_byok_oauth_router)
app.include_router(anthropic_router)
app.include_router(anthropic_skills_router)
app.include_router(evals_router)
app.include_router(claude_code_marketplace_router)
app.include_router(google_router)
app.include_router(langfuse_router)
app.include_router(pass_through_router)
app.include_router(health_router)
app.include_router(key_management_router)
app.include_router(internal_user_router)
app.include_router(team_router)
app.include_router(ui_sso_router)
app.include_router(scim_router)
app.include_router(organization_router)
app.include_router(customer_router)
app.include_router(spend_management_router)
app.include_router(cloudzero_router)
app.include_router(vantage_router)
app.include_router(caching_router)
app.include_router(analytics_router)
app.include_router(guardrails_router)
app.include_router(policy_router)
app.include_router(usage_ai_router)
app.include_router(policy_crud_router)
app.include_router(policy_resolve_router)
app.include_router(search_tool_management_router)
app.include_router(prompts_router)
app.include_router(callback_management_endpoints_router)
app.include_router(debugging_endpoints_router)
app.include_router(ui_crud_endpoints_router)
app.include_router(openai_files_router)
app.include_router(team_callback_router)
app.include_router(jwt_key_mapping_router)
app.include_router(budget_management_router)
app.include_router(model_management_router)
app.include_router(model_access_group_management_router)
app.include_router(tag_management_router)
app.include_router(tool_management_router)
app.include_router(workflow_management_router)
app.include_router(memory_router)
app.include_router(cost_tracking_settings_router)
app.include_router(router_settings_router)
app.include_router(fallback_management_router)
app.include_router(cache_settings_router)
app.include_router(config_override_router)
app.include_router(user_agent_analytics_router)
app.include_router(enterprise_router)
app.include_router(ui_discovery_endpoints_router)
app.include_router(agent_endpoints_router)
app.include_router(compliance_router)
app.include_router(a2a_router)
app.include_router(access_group_router)
# Eager: /models/{name}:method overlaps with the OpenAI /models endpoint.
app.include_router(google_router)
attach_lazy_features(app)
async def _stream_mcp_asgi_response(
@ -14690,8 +14685,3 @@ async def dynamic_mcp_route(mcp_server_name: str, request: Request):
f"Error handling dynamic MCP route for {mcp_server_name}: {str(e)}"
)
raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
app.mount(path=BASE_MCP_ROUTE, app=mcp_app)
app.include_router(mcp_rest_endpoints_router)
app.include_router(mcp_discoverable_endpoints_router)

2
tests/litellm_utils_tests/test_utils.py
View file

@ -812,7 +812,7 @@ def test_redact_msgs_from_logs_with_dynamic_params():
# Assert redaction occurred
assert _redacted_response_obj.choices[0].message.content == "redacted-by-litellm"
# Test Case 3: standard_callback_dynamic_params does not override litellm.turn_off_message_logging
# Test Case 3: standard_callback_dynamic_params does not set turn_off_message_logging
# since litellm.turn_off_message_logging is True redaction should occur
standard_callback_dynamic_params = StandardCallbackDynamicParams()
litellm_logging_obj.model_call_details["standard_callback_dynamic_params"] = (

78
tests/logging_callback_tests/test_logging_redaction_e2e_test.py
View file

@ -13,11 +13,13 @@ import logging
import time
from unittest.mock import AsyncMock, patch
import httpx
import pytest
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.responses.main import mock_responses_api_response
from litellm.types.utils import StandardLoggingPayload
@ -126,17 +128,10 @@ async def test_redaction_responses_api():
test_custom_logger = TestCustomLogger(turn_off_message_logging=True)
litellm.callbacks = [test_custom_logger]
# Mock a ResponsesAPIResponse-style response
mock_response = {
"output": [{"text": "This is a test response"}],
"model": "gpt-3.5-turbo",
"usage": {"input_tokens": 5, "output_tokens": 5, "total_tokens": 10},
}
response = await litellm.aresponses(
model="gpt-3.5-turbo",
input="hi",
mock_response=mock_response,
mock_response="This is a test response",
)
await asyncio.sleep(1)
@ -163,6 +158,7 @@ async def test_redaction_responses_api():
assert (
content_item["text"] == "redacted-by-litellm"
), f"Expected redacted text but got: {content_item['text']}"
assert "This is a test response" not in json.dumps(standard_logging_payload)
print(
"logged standard logging payload for ResponsesAPIResponse",
json.dumps(standard_logging_payload, indent=2),
@ -176,29 +172,36 @@ async def test_redaction_responses_api_stream():
test_custom_logger = TestCustomLogger(turn_off_message_logging=True)
litellm.callbacks = [test_custom_logger]
# Mock a ResponsesAPIResponse-style response with streaming chunks
mock_response = [
{
"output": [{"text": "This"}],
"model": "gpt-3.5-turbo",
},
{
"output": [{"text": " is"}],
"model": "gpt-3.5-turbo",
},
{
"output": [{"text": " a test response"}],
"model": "gpt-3.5-turbo",
"usage": {"input_tokens": 5, "output_tokens": 5, "total_tokens": 10},
},
]
mocked_response_payload = mock_responses_api_response(
"This is a test response"
).model_dump()
response = await litellm.aresponses(
model="gpt-3.5-turbo",
input="hi",
mock_response=mock_response,
stream=True,
)
async def mock_post(self, url, headers, timeout, stream=False, **kwargs):
stream_content = (
"data: "
+ json.dumps(
{
"type": "response.completed",
"response": mocked_response_payload,
}
)
+ "\n\ndata: [DONE]\n\n"
)
return httpx.Response(
status_code=200,
content=stream_content,
request=httpx.Request("POST", url),
)
with patch(
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
new=mock_post,
):
response = await litellm.aresponses(
model="gpt-3.5-turbo",
input="hi",
stream=True,
)
# Consume the stream
chunks = []
@ -445,18 +448,11 @@ async def test_disable_redaction_header_responses_api():
test_custom_logger = TestCustomLogger()
litellm.callbacks = [test_custom_logger]
# Mock a ResponsesAPIResponse-style response
mock_response = {
"output": [{"text": "This is a test response"}],
"model": "gpt-3.5-turbo",
"usage": {"input_tokens": 5, "output_tokens": 5, "total_tokens": 10},
}
# Pass the header via litellm_metadata (as the proxy does for Responses API)
response = await litellm.aresponses(
model="gpt-3.5-turbo",
input="hi",
mock_response=mock_response,
mock_response="This is a test response",
litellm_metadata={"headers": {"litellm-disable-message-redaction": "true"}},
)
@ -464,14 +460,14 @@ async def test_disable_redaction_header_responses_api():
standard_logging_payload = test_custom_logger.logged_standard_logging_payload
assert standard_logging_payload is not None
# Verify that messages are NOT redacted because the header was set
# Verify that the direct SDK path still honors the explicit header.
print(
"logged standard logging payload for ResponsesAPI with disable header",
json.dumps(standard_logging_payload, indent=2, default=str),
)
# The content should NOT be redacted
assert standard_logging_payload["response"] != {"text": "redacted-by-litellm"}
response = standard_logging_payload["response"]
assert response["output"][0]["content"][0]["text"] == "This is a test response"
assert standard_logging_payload["messages"][0]["content"] == "hi"

120
tests/proxy_unit_tests/test_audit_logs_proxy.py
View file

@ -45,8 +45,11 @@ verbose_proxy_logger.setLevel(level=logging.DEBUG)
from starlette.datastructures import URL
from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update
from litellm.proxy._types import LiteLLM_AuditLogs, LitellmTableNames
from litellm.proxy.management_helpers.audit_logs import (
create_audit_log_for_update,
get_audit_log_changed_by,
)
from litellm.proxy._types import LiteLLM_AuditLogs, LitellmTableNames, UserAPIKeyAuth
from litellm.caching.caching import DualCache
from unittest.mock import patch, AsyncMock
@ -54,6 +57,119 @@ proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
import json
def test_get_audit_log_changed_by_prefers_authenticated_user():
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
user_id="authenticated-user",
)
assert (
get_audit_log_changed_by(
litellm_changed_by="spoofed-user",
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name="proxy-admin",
)
== "authenticated-user"
)
def test_get_audit_log_changed_by_honors_header_with_admin_opt_in():
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
user_id="service-account",
metadata={"allow_litellm_changed_by_header": True},
)
assert (
get_audit_log_changed_by(
litellm_changed_by="delegated-user",
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name="proxy-admin",
)
== "delegated-user"
)
def test_get_audit_log_changed_by_honors_header_with_team_opt_in():
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
user_id="service-account",
team_metadata={"allow_litellm_changed_by_header": True},
)
assert (
get_audit_log_changed_by(
litellm_changed_by="delegated-user",
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name="proxy-admin",
)
== "delegated-user"
)
def test_get_audit_log_changed_by_ignores_header_without_opt_in_when_user_id_missing():
user_api_key_dict = UserAPIKeyAuth(api_key="test-key")
assert (
get_audit_log_changed_by(
litellm_changed_by="spoofed-user",
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name="proxy-admin",
)
== "proxy-admin"
)
def test_get_audit_log_changed_by_honors_header_with_opt_in_when_user_id_missing():
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
metadata={"allow_litellm_changed_by_header": True},
)
assert (
get_audit_log_changed_by(
litellm_changed_by="delegated-user",
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name="proxy-admin",
)
== "delegated-user"
)
@pytest.mark.asyncio
async def test_create_internal_user_audit_log_uses_changed_by_helper():
from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
user_id="service-account",
metadata={"allow_litellm_changed_by_header": True},
)
with (
patch("litellm.store_audit_logs", True),
patch(
"litellm.proxy.hooks.user_management_event_hooks.create_audit_log_for_update",
new_callable=AsyncMock,
) as mock_create_audit_log_for_update,
):
await UserManagementEventHooks.create_internal_user_audit_log(
user_id="target-user",
action="updated",
litellm_changed_by="delegated-user",
user_api_key_dict=user_api_key_dict,
litellm_proxy_admin_name="proxy-admin",
before_value='{"before": true}',
after_value='{"after": true}',
)
request_data = mock_create_audit_log_for_update.await_args.kwargs["request_data"]
assert request_data.changed_by == "delegated-user"
assert request_data.changed_by_api_key == "test-key"
assert request_data.object_id == "target-user"
assert request_data.action == "updated"
@pytest.mark.asyncio
async def test_create_audit_log_for_update_premium_user():
"""

17
tests/proxy_unit_tests/test_proxy_routes.py
View file

@ -39,6 +39,23 @@ def test_routes_on_litellm_proxy():
this prevents accidentelly deleting /threads, or /batches etc
"""
# Force-load lazy features so the test sees the full route set. Continue
# on per-feature import failure — the assertion below still catches
# missing-route regressions.
import importlib
from litellm.proxy._lazy_features import LAZY_FEATURES
registered_paths = [getattr(r, "path", "") for r in app.routes]
for feat in LAZY_FEATURES:
if any(rp.startswith(p) for p in feat.path_prefixes for rp in registered_paths):
continue
try:
module = importlib.import_module(feat.module_path)
feat.register_fn(app, module)
except Exception as exc:
print(f"warning: failed to force-load {feat.name}: {exc}")
_all_routes = []
for route in app.routes:

12
tests/proxy_unit_tests/test_proxy_server.py
View file

@ -1553,6 +1553,7 @@ async def test_add_callback_via_key(prisma_client):
fastapi_response=Response(),
user_api_key_dict=UserAPIKeyAuth(
metadata={
"allow_client_mock_response": True,
"logging": [
{
"callback_name": "langfuse", # 'otel', 'langfuse', 'lunary'
@ -1563,7 +1564,7 @@ async def test_add_callback_via_key(prisma_client):
"langfuse_host": "https://us.cloud.langfuse.com",
},
}
]
],
}
),
)
@ -1657,6 +1658,7 @@ async def test_add_callback_via_key_litellm_pre_call_utils(
team_id=None,
max_parallel_requests=None,
metadata={
"allow_client_mock_response": True,
"logging": [
{
"callback_name": "langfuse",
@ -1667,7 +1669,7 @@ async def test_add_callback_via_key_litellm_pre_call_utils(
"langfuse_host": "https://us.cloud.langfuse.com",
},
}
]
],
},
tpm_limit=None,
rpm_limit=None,
@ -1813,6 +1815,7 @@ async def test_add_callback_via_key_litellm_pre_call_utils_gcs_bucket(
team_id=None,
max_parallel_requests=None,
metadata={
"allow_client_mock_response": True,
"logging": [
{
"callback_name": "gcs_bucket",
@ -1822,7 +1825,7 @@ async def test_add_callback_via_key_litellm_pre_call_utils_gcs_bucket(
"gcs_path_service_account": "pathrise-convert-1606954137718-a956eef1a2a8.json",
},
}
]
],
},
tpm_limit=None,
rpm_limit=None,
@ -1946,6 +1949,7 @@ async def test_add_callback_via_key_litellm_pre_call_utils_langsmith(
team_id=None,
max_parallel_requests=None,
metadata={
"allow_client_mock_response": True,
"logging": [
{
"callback_name": "langsmith",
@ -1956,7 +1960,7 @@ async def test_add_callback_via_key_litellm_pre_call_utils_langsmith(
"langsmith_base_url": "https://api.smith.langchain.com",
},
}
]
],
},
tpm_limit=None,
rpm_limit=None,

52
tests/proxy_unit_tests/test_proxy_utils.py
View file

@ -235,18 +235,10 @@ async def test_add_key_or_team_level_spend_logs_metadata_to_request(
"langfuse_host": "https://us.cloud.langfuse.com",
"langfuse_public_key": "pk-lf-9636b7a6-c066",
"langfuse_secret_key": "sk-lf-7cc8b620",
},
{
"langfuse_host": "os.environ/LANGFUSE_HOST_TEMP",
"langfuse_public_key": "os.environ/LANGFUSE_PUBLIC_KEY_TEMP",
"langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY_TEMP",
},
}
],
)
def test_dynamic_logging_metadata_key_and_team_metadata(callback_vars):
os.environ["LANGFUSE_PUBLIC_KEY_TEMP"] = "pk-lf-9636b7a6-c066"
os.environ["LANGFUSE_SECRET_KEY_TEMP"] = "sk-lf-7cc8b620"
os.environ["LANGFUSE_HOST_TEMP"] = "https://us.cloud.langfuse.com"
from litellm.proxy.proxy_server import ProxyConfig
proxy_config = ProxyConfig()
@ -317,6 +309,41 @@ def test_dynamic_logging_metadata_key_and_team_metadata(callback_vars):
assert "os.environ" not in var
def test_dynamic_logging_metadata_ignores_env_references_from_key_metadata(
monkeypatch,
):
monkeypatch.setenv("LANGFUSE_SECRET_KEY_TEMP", "server-side-secret")
monkeypatch.setattr(
litellm.utils,
"get_secret",
lambda *args, **kwargs: pytest.fail("get_secret should not be called"),
)
from litellm.proxy.proxy_server import ProxyConfig
proxy_config = ProxyConfig()
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
metadata={
"logging": [
{
"callback_name": "langfuse",
"callback_type": "success",
"callback_vars": {
"langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY_TEMP",
},
}
]
},
team_metadata={},
)
callbacks = _get_dynamic_logging_metadata(
user_api_key_dict=user_api_key_dict, proxy_config=proxy_config
)
assert callbacks is None
@pytest.mark.parametrize(
"callback_vars",
[
@ -1263,11 +1290,16 @@ def test_proxy_config_state_post_init_callback_call(monkeypatch):
}
)
LiteLLMProxyRequestSetup.add_team_based_callbacks_from_config(
callback_metadata = LiteLLMProxyRequestSetup.add_team_based_callbacks_from_config(
team_id="test",
proxy_config=pc,
)
assert callback_metadata is not None
assert callback_metadata.callback_vars is not None
assert callback_metadata.callback_vars["langfuse_public_key"] == "test_public_key"
assert callback_metadata.callback_vars["langfuse_secret"] == "test_secret_key"
config = pc.get_config_state()
assert config["litellm_settings"]["default_team_settings"][0]["team_id"] == "test"

210
tests/test_litellm/litellm_core_utils/test_redact_messages.py
View file

@ -5,10 +5,17 @@ Covers the proxy flow where headers arrive in litellm_params["metadata"]["header
but litellm_params["litellm_metadata"] is None.
"""
from types import SimpleNamespace
import pytest
import litellm
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.litellm_core_utils.redact_messages import (
_redact_responses_api_output,
perform_redaction,
should_redact_message_logging,
)
from litellm.responses.main import mock_responses_api_response
@pytest.fixture(autouse=True)
@ -68,8 +75,7 @@ class TestShouldRedactMessageLogging:
assert should_redact_message_logging(details) is True
def test_disable_redaction_via_header_proxy_flow(self):
"""litellm-disable-message-redaction should suppress redaction
even when global setting is on, and litellm_metadata is None."""
"""Core helper still honors the explicit disable-redaction header."""
litellm.turn_off_message_logging = True
details = _make_model_call_details(
metadata_headers={"litellm-disable-message-redaction": "true"},
@ -77,6 +83,14 @@ class TestShouldRedactMessageLogging:
)
assert should_redact_message_logging(details) is False
def test_disable_redaction_via_header_when_global_off(self):
"""litellm-disable-message-redaction is still honored when global redaction is off."""
details = _make_model_call_details(
metadata_headers={"litellm-disable-message-redaction": "true"},
litellm_metadata=None,
)
assert should_redact_message_logging(details) is False
# ---- SDK direct-call flow: headers in litellm_metadata ----
def test_enable_redaction_via_header_in_litellm_metadata(self):
@ -127,6 +141,16 @@ class TestShouldRedactMessageLogging:
)
assert should_redact_message_logging(details) is False
def test_dynamic_param_false_overrides_global_redaction(self):
"""Dynamic turn_off_message_logging=False should take precedence."""
litellm.turn_off_message_logging = True
details = _make_model_call_details(
metadata_headers={},
litellm_metadata=None,
standard_callback_dynamic_params={"turn_off_message_logging": False},
)
assert should_redact_message_logging(details) is False
# ---- non-dict metadata safety ----
def test_both_metadata_fields_none(self):
@ -145,3 +169,183 @@ class TestShouldRedactMessageLogging:
litellm_metadata=None,
)
assert should_redact_message_logging(details) is True
class TestPerformRedaction:
def test_redacts_standard_logging_and_responses_api_dicts(self):
details = {
"messages": [{"role": "user", "content": "sensitive input"}],
"prompt": "sensitive prompt",
"input": "sensitive input",
"standard_logging_object": {
"messages": [{"role": "user", "content": "sensitive input"}],
"response": {
"output": [
{"text": "top-level text"},
{"content": [{"text": "nested text"}]},
{"type": "reasoning", "summary": [{"text": "reasoning"}]},
],
"usage": {"total_tokens": 1},
},
},
}
result = {
"output": [
{"text": "top-level result"},
{"content": [{"text": "nested result"}]},
{"type": "reasoning", "summary": [{"text": "reasoning result"}]},
],
"usage": {"total_tokens": 1},
}
redacted = perform_redaction(details, result)
assert details["messages"] == [
{"role": "user", "content": "redacted-by-litellm"}
]
assert details["prompt"] == ""
assert details["input"] == ""
logged_response = details["standard_logging_object"]["response"]
assert logged_response["usage"] == {"total_tokens": 1}
assert logged_response["output"][0]["text"] == "redacted-by-litellm"
assert logged_response["output"][1]["content"][0]["text"] == (
"redacted-by-litellm"
)
assert logged_response["output"][2]["summary"][0]["text"] == (
"redacted-by-litellm"
)
assert redacted["usage"] == {"total_tokens": 1}
assert redacted["output"][0]["text"] == "redacted-by-litellm"
assert redacted["output"][1]["content"][0]["text"] == "redacted-by-litellm"
assert redacted["output"][2]["summary"][0]["text"] == "redacted-by-litellm"
assert result["output"][0]["text"] == "top-level result"
def test_redacts_model_response_dict_choices(self):
result = {
"choices": [
{
"message": {
"content": "message content",
"reasoning_content": "message reasoning",
"thinking_blocks": ["thinking"],
"audio": {"data": "audio"},
}
},
{
"delta": {
"content": "delta content",
"reasoning_content": "delta reasoning",
"thinking_blocks": ["delta thinking"],
"audio": {"data": "audio"},
}
},
]
}
redacted = perform_redaction({}, result)
message = redacted["choices"][0]["message"]
assert message["content"] == "redacted-by-litellm"
assert message["reasoning_content"] == "redacted-by-litellm"
assert message["thinking_blocks"] is None
assert message["audio"] is None
delta = redacted["choices"][1]["delta"]
assert delta["content"] == "redacted-by-litellm"
assert delta["reasoning_content"] == "redacted-by-litellm"
assert delta["thinking_blocks"] is None
assert delta["audio"] is None
def test_redacts_standard_logging_model_response_dict_choices(self):
details = {
"standard_logging_object": {
"response": {
"choices": [
{
"message": {
"content": "message content",
"reasoning_content": "message reasoning",
"thinking_blocks": ["thinking"],
"audio": {"data": "audio"},
}
},
{
"delta": {
"content": "delta content",
"reasoning_content": "delta reasoning",
"thinking_blocks": ["delta thinking"],
"audio": {"data": "audio"},
}
},
]
}
}
}
perform_redaction(details, None)
choices = details["standard_logging_object"]["response"]["choices"]
message = choices[0]["message"]
assert message["content"] == "redacted-by-litellm"
assert message["reasoning_content"] == "redacted-by-litellm"
assert message["thinking_blocks"] is None
assert message["audio"] is None
delta = choices[1]["delta"]
assert delta["content"] == "redacted-by-litellm"
assert delta["reasoning_content"] == "redacted-by-litellm"
assert delta["thinking_blocks"] is None
assert delta["audio"] is None
def test_redacts_object_choices_inside_model_response_dict(self):
result = {
"choices": [
litellm.Choices(
message=litellm.Message(
content="message content",
role="assistant",
reasoning_content="message reasoning",
)
)
]
}
redacted = perform_redaction({}, result)
choice = redacted["choices"][0]
assert choice.message.content == "redacted-by-litellm"
assert choice.message.reasoning_content == "redacted-by-litellm"
def test_redacts_response_output_objects_with_top_level_text(self):
output_items = [
SimpleNamespace(text="top-level output"),
"non-dict output item",
]
_redact_responses_api_output(output_items)
assert output_items[0].text == "redacted-by-litellm"
assert output_items[1] == "non-dict output item"
def test_skips_non_dict_response_output_items(self):
result = {
"output": [
"non-dict output item",
{"content": [{"text": "nested result"}]},
]
}
redacted = perform_redaction({}, result)
assert redacted["output"][0] == "non-dict output item"
assert redacted["output"][1]["content"][0]["text"] == "redacted-by-litellm"
def test_redacts_responses_api_response_object(self):
response = mock_responses_api_response("sensitive output")
redacted = perform_redaction({}, response)
assert redacted.output[0].content[0].text == "redacted-by-litellm"
assert response.output[0].content[0].text == "sensitive output"

54
tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py
View file

@ -1853,6 +1853,60 @@ async def test_make_bedrock_api_request_logging_event_type_for_spend_logs():
assert mock_log.call_args.kwargs["event_type"] == GuardrailEventHooks.pre_call
@pytest.mark.asyncio
async def test_make_bedrock_api_request_filters_dynamic_evaluation_overrides():
guardrail = BedrockGuardrail(
guardrailIdentifier="test-guardrail", guardrailVersion="DRAFT"
)
mock_credentials = MagicMock()
mock_credentials.access_key = "test-access-key"
mock_credentials.secret_key = "test-secret-key"
mock_credentials.token = None
mock_bedrock_response = MagicMock()
mock_bedrock_response.status_code = 200
mock_bedrock_response.json.return_value = {"action": "NONE", "assessments": []}
prepared_request = MagicMock()
prepared_request.url = "https://bedrock.test/apply"
prepared_request.body = b"{}"
prepared_request.headers = {}
with (
patch.object(
guardrail.async_handler, "post", new_callable=AsyncMock
) as mock_post,
patch.object(
guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1")
),
patch.object(
guardrail, "_prepare_request", return_value=prepared_request
) as mock_prepare_request,
patch.object(
guardrail,
"get_guardrail_dynamic_request_body_params",
return_value={
"content": [{"text": {"text": "benign replacement"}}],
"source": "OUTPUT",
"outputScope": "FULL",
},
),
):
mock_post.return_value = mock_bedrock_response
await guardrail.make_bedrock_api_request(
source="INPUT",
messages=[{"role": "user", "content": "actual prompt"}],
request_data={"model": "gpt-4o"},
)
prepared_data = mock_prepare_request.call_args.kwargs["data"]
assert prepared_data["source"] == "INPUT"
assert "actual prompt" in json.dumps(prepared_data["content"])
assert "benign replacement" not in json.dumps(prepared_data["content"])
assert prepared_data["outputScope"] == "FULL"
@pytest.mark.asyncio
async def test_during_call_hook_invokes_bedrock_async_moderation_hook():
"""

32
tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py
View file

@ -505,6 +505,38 @@ def test_get_logging_caching_headers_pillar_metadata():
)
def test_get_logging_caching_headers_ignores_untrusted_pillar_headers():
request_data = {
"metadata": {
"pillar_response_headers": {
"set-cookie": "session=evil",
"x-pillar-flagged": "true",
},
"pillar_flagged": True,
}
}
headers = get_logging_caching_headers(request_data)
assert "set-cookie" not in headers
assert "x-pillar-flagged" not in headers
def test_get_logging_caching_headers_filters_non_pillar_headers():
request_data = {
"metadata": {
"pillar_flagged": True,
}
}
build_pillar_response_headers(request_data["metadata"])
request_data["metadata"]["pillar_response_headers"]["set-cookie"] = "session=evil"
headers = get_logging_caching_headers(request_data)
assert headers["x-pillar-flagged"] == "true"
assert "set-cookie" not in headers
def test_get_logging_caching_headers_truncates_large_evidence():
long_text = "悪" * 6000 # multi-byte unicode to test URL encoding and truncation
request_data = {

341
tests/test_litellm/proxy/test_litellm_pre_call_utils.py
View file

@ -7,10 +7,11 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import Request
from pydantic import ValidationError as PydanticValidationError
from starlette.datastructures import Headers
import litellm
from litellm.proxy._types import TeamCallbackMetadata, UserAPIKeyAuth
from litellm.proxy._types import AddTeamCallback, TeamCallbackMetadata, UserAPIKeyAuth
from litellm.proxy.litellm_pre_call_utils import (
KeyAndTeamLoggingSettings,
LiteLLMProxyRequestSetup,
@ -512,6 +513,247 @@ async def test_add_litellm_data_to_request_strips_string_encoded_admin_injection
assert "_pipeline_managed_guardrails" not in other
@pytest.mark.asyncio
async def test_add_litellm_data_to_request_strips_user_control_fields():
"""Strip untrusted proxy-control fields before guardrails, logging, and headers read metadata."""
request_mock = MagicMock(spec=Request)
request_mock.url.path = "/v1/chat/completions"
request_mock.url = MagicMock()
request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions"
request_mock.method = "POST"
request_mock.query_params = {}
request_mock.headers = {"Content-Type": "application/json"}
request_mock.client = MagicMock()
request_mock.client.host = "127.0.0.1"
malicious_metadata = {
"disable_global_guardrails": True,
"opted_out_global_guardrails": ["pii"],
"pillar_response_headers": {"set-cookie": "session=evil"},
"_pillar_response_headers_trusted": True,
"pillar_flagged": True,
"pillar_scanners": {"jailbreak": True},
"pillar_evidence": [{"evidence": "spoofed"}],
"pillar_session_id_response": "spoofed-session",
"applied_guardrails": ["spoofed"],
"applied_policies": ["spoofed-policy"],
"policy_sources": {"spoofed-policy": "request"},
"_guardrail_pipelines": [{"name": "spoofed"}],
"_pipeline_managed_guardrails": ["evaded"],
"safe_user_metadata": "kept",
}
data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "hello"}],
"mock_response": "free response",
"mock_tool_calls": [{"id": "call_1"}],
"disable_global_guardrails": True,
"metadata": copy.deepcopy(malicious_metadata),
"litellm_metadata": copy.deepcopy(malicious_metadata),
}
updated = await add_litellm_data_to_request(
data=data,
request=request_mock,
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
proxy_config=MagicMock(),
general_settings={},
version="test-version",
)
assert "mock_response" not in updated
assert "mock_tool_calls" not in updated
assert "disable_global_guardrails" not in updated
stripped_keys = {
"disable_global_guardrails",
"opted_out_global_guardrails",
"pillar_response_headers",
"_pillar_response_headers_trusted",
"pillar_flagged",
"pillar_scanners",
"pillar_evidence",
"pillar_session_id_response",
"applied_guardrails",
"applied_policies",
"policy_sources",
"_guardrail_pipelines",
"_pipeline_managed_guardrails",
}
for metadata_key in ("metadata", "litellm_metadata"):
cleaned_metadata = updated.get(metadata_key) or {}
for stripped_key in stripped_keys:
assert stripped_key not in cleaned_metadata
assert cleaned_metadata.get("safe_user_metadata") == "kept"
requester_metadata = updated["metadata"]["requester_metadata"]
for stripped_key in stripped_keys:
assert stripped_key not in requester_metadata
snapshot_body = updated["proxy_server_request"]["body"]
assert "mock_response" not in snapshot_body
assert "mock_tool_calls" not in snapshot_body
assert "pillar_response_headers" not in snapshot_body["metadata"]
@pytest.mark.asyncio
async def test_add_litellm_data_to_request_allows_client_mock_response_with_admin_opt_in():
request_mock = MagicMock(spec=Request)
request_mock.url.path = "/v1/chat/completions"
request_mock.url = MagicMock()
request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions"
request_mock.method = "POST"
request_mock.query_params = {}
request_mock.headers = {"Content-Type": "application/json"}
request_mock.client = MagicMock()
request_mock.client.host = "127.0.0.1"
updated = await add_litellm_data_to_request(
data={
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "hello"}],
"mock_response": "allowed mock",
"mock_tool_calls": [{"id": "call_1"}],
},
request=request_mock,
user_api_key_dict=UserAPIKeyAuth(
api_key="hashed-key",
metadata={"allow_client_mock_response": True},
),
proxy_config=MagicMock(),
general_settings={},
version="test-version",
)
assert updated["mock_response"] == "allowed mock"
assert updated["mock_tool_calls"] == [{"id": "call_1"}]
@pytest.mark.asyncio
async def test_add_litellm_data_to_request_strips_client_redaction_bypass_controls():
request_mock = MagicMock(spec=Request)
request_mock.url.path = "/v1/chat/completions"
request_mock.url = MagicMock()
request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions"
request_mock.method = "POST"
request_mock.query_params = {}
request_mock.headers = {
"Content-Type": "application/json",
"litellm-disable-message-redaction": "true",
}
request_mock.client = MagicMock()
request_mock.client.host = "127.0.0.1"
original_turn_off_message_logging = litellm.turn_off_message_logging
litellm.turn_off_message_logging = True
try:
updated = await add_litellm_data_to_request(
data={
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "hello"}],
"turn_off_message_logging": False,
"metadata": {"headers": {"litellm-disable-message-redaction": "true"}},
"litellm_metadata": json.dumps(
{"headers": {"LiteLLM-Disable-Message-Redaction": "true"}}
),
},
request=request_mock,
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
proxy_config=MagicMock(),
general_settings={},
version="test-version",
)
finally:
litellm.turn_off_message_logging = original_turn_off_message_logging
assert "turn_off_message_logging" not in updated
assert "litellm-disable-message-redaction" not in {
header.lower() for header in updated["metadata"]["headers"]
}
assert "litellm-disable-message-redaction" not in {
header.lower()
for header in updated["metadata"]["requester_metadata"].get("headers", {})
}
assert "litellm-disable-message-redaction" not in {
header.lower() for header in updated["proxy_server_request"]["headers"]
}
assert "litellm-disable-message-redaction" not in {
header.lower()
for header in updated["proxy_server_request"]["body"]["metadata"]["headers"]
}
assert "litellm-disable-message-redaction" not in {
header.lower()
for header in (updated.get("litellm_metadata") or {}).get("headers", {})
}
@pytest.mark.parametrize(
"auth_kwargs",
[
{"metadata": {"allow_client_message_redaction_opt_out": True}},
{"team_metadata": {"allow_client_message_redaction_opt_out": True}},
],
)
@pytest.mark.asyncio
async def test_add_litellm_data_to_request_allows_redaction_opt_out_with_admin_opt_in(
auth_kwargs,
):
request_mock = MagicMock(spec=Request)
request_mock.url.path = "/v1/chat/completions"
request_mock.url = MagicMock()
request_mock.url.__str__.return_value = "http://localhost/v1/chat/completions"
request_mock.method = "POST"
request_mock.query_params = {}
request_mock.headers = {
"Content-Type": "application/json",
"litellm-disable-message-redaction": "true",
}
request_mock.client = MagicMock()
request_mock.client.host = "127.0.0.1"
original_turn_off_message_logging = litellm.turn_off_message_logging
litellm.turn_off_message_logging = True
try:
updated = await add_litellm_data_to_request(
data={
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "hello"}],
"turn_off_message_logging": False,
"metadata": {"headers": {"litellm-disable-message-redaction": "true"}},
"litellm_metadata": json.dumps(
{"headers": {"LiteLLM-Disable-Message-Redaction": "true"}}
),
},
request=request_mock,
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", **auth_kwargs),
proxy_config=MagicMock(),
general_settings={},
version="test-version",
)
finally:
litellm.turn_off_message_logging = original_turn_off_message_logging
assert updated["turn_off_message_logging"] is False
assert "litellm-disable-message-redaction" in {
header.lower() for header in updated["metadata"]["headers"]
}
assert "litellm-disable-message-redaction" in {
header.lower()
for header in updated["metadata"]["requester_metadata"].get("headers", {})
}
assert "litellm-disable-message-redaction" in {
header.lower() for header in updated["proxy_server_request"]["headers"]
}
assert "litellm-disable-message-redaction" in {
header.lower()
for header in updated["proxy_server_request"]["body"]["metadata"]["headers"]
}
assert "litellm-disable-message-redaction" in {
header.lower()
for header in (updated.get("litellm_metadata") or {}).get("headers", {})
}
@pytest.mark.asyncio
async def test_add_litellm_data_to_request_ignores_x_litellm_tags_header_without_permission():
"""Regression: the `x-litellm-tags` header bypassed the body-metadata
@ -1274,6 +1516,51 @@ def test_get_dynamic_logging_metadata_with_arize_team_logging():
assert result.callback_vars["arize_space_id"] == "test_arize_space_id"
def test_add_team_callback_rejects_env_reference():
with pytest.raises(PydanticValidationError) as exc_info:
AddTeamCallback(
callback_name="langfuse",
callback_type="success",
callback_vars={
"langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY_TEMP"
},
)
assert "os.environ/" in str(exc_info.value)
def test_get_dynamic_logging_metadata_ignores_env_reference_from_key_metadata(
monkeypatch,
):
monkeypatch.setenv("LANGFUSE_SECRET_KEY_TEMP", "server-side-secret")
monkeypatch.setattr(
litellm.utils,
"get_secret",
lambda *args, **kwargs: pytest.fail("get_secret should not be called"),
)
user_api_key_dict = UserAPIKeyAuth(
api_key="test-key",
metadata={
"logging": [
{
"callback_name": "langfuse",
"callback_type": "success",
"callback_vars": {
"langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY_TEMP",
},
}
]
},
team_metadata={},
)
result = _get_dynamic_logging_metadata(
user_api_key_dict=user_api_key_dict, proxy_config=MagicMock()
)
assert result is None
def test_get_num_retries_from_request():
"""
Test LiteLLMProxyRequestSetup._get_num_retries_from_request method
@ -1669,7 +1956,10 @@ async def test_add_litellm_metadata_from_request_headers():
# Create mock user API key dict
mock_user_api_key_dict = UserAPIKeyAuth(
api_key="test-key", user_id="test-user", org_id="test-org"
api_key="test-key",
user_id="test-user",
org_id="test-org",
metadata={"allow_client_mock_response": True},
)
# Create mock proxy logging object
@ -1782,7 +2072,9 @@ async def test_anthropic_messages_standard_logging_object_matches_fixture():
mock_fastapi_response = MagicMock(spec=Response)
mock_user_api_key_dict = UserAPIKeyAuth(
api_key="test-key", user_id="default_user_id"
api_key="test-key",
user_id="default_user_id",
metadata={"allow_client_mock_response": True},
)
mock_proxy_logging_obj = MagicMock(spec=ProxyLogging)
@ -3326,7 +3618,9 @@ async def test_team_guardrail_merges_with_global_policy():
policy_registry = get_policy_registry()
policy_registry._policies = {
"global-policy": Policy(
guardrails=PolicyGuardrails(add=["policy-guardrail-1", "policy-guardrail-2"]),
guardrails=PolicyGuardrails(
add=["policy-guardrail-1", "policy-guardrail-2"]
),
),
}
policy_registry._initialized = True
@ -3347,14 +3641,18 @@ async def test_team_guardrail_merges_with_global_policy():
guardrails = data["metadata"].get("guardrails", [])
assert "team-direct-guardrail" in guardrails, \
f"Team guardrail missing from merged list: {guardrails}"
assert "policy-guardrail-1" in guardrails, \
f"policy-guardrail-1 missing: {guardrails}"
assert "policy-guardrail-2" in guardrails, \
f"policy-guardrail-2 missing: {guardrails}"
assert len(guardrails) == len(set(guardrails)), \
f"Duplicates in guardrails list: {guardrails}"
assert (
"team-direct-guardrail" in guardrails
), f"Team guardrail missing from merged list: {guardrails}"
assert (
"policy-guardrail-1" in guardrails
), f"policy-guardrail-1 missing: {guardrails}"
assert (
"policy-guardrail-2" in guardrails
), f"policy-guardrail-2 missing: {guardrails}"
assert len(guardrails) == len(
set(guardrails)
), f"Duplicates in guardrails list: {guardrails}"
# Verify get_guardrail_from_metadata returns the merged list even
# when litellm_metadata is present (the bug: it returned [] before fix)
@ -3365,9 +3663,9 @@ async def test_team_guardrail_merges_with_global_policy():
dummy = _DummyGuardrail(guardrail_name="team-direct-guardrail")
returned = dummy.get_guardrail_from_metadata(data)
assert "team-direct-guardrail" in returned, (
f"get_guardrail_from_metadata shadowed by litellm_metadata; got: {returned}"
)
assert (
"team-direct-guardrail" in returned
), f"get_guardrail_from_metadata shadowed by litellm_metadata; got: {returned}"
finally:
policy_registry._policies = {}
@ -3396,9 +3694,10 @@ async def test_get_guardrail_from_metadata_prefers_metadata_over_litellm_metadat
}
result = dummy.get_guardrail_from_metadata(data)
assert result == ["my-guardrail", "other-guardrail"], (
f"Expected guardrails from metadata, got: {result}"
)
assert result == [
"my-guardrail",
"other-guardrail",
], f"Expected guardrails from metadata, got: {result}"
def test_get_guardrail_from_metadata_reads_litellm_metadata_when_no_metadata():
@ -3419,6 +3718,6 @@ def test_get_guardrail_from_metadata_reads_litellm_metadata_when_no_metadata():
}
result = dummy.get_guardrail_from_metadata(data)
assert result == ["my-guardrail"], (
f"Expected guardrails from litellm_metadata fallback, got: {result}"
)
assert result == [
"my-guardrail"
], f"Expected guardrails from litellm_metadata fallback, got: {result}"

252
tests/test_litellm/proxy/test_proxy_server.py
View file

@ -5471,3 +5471,255 @@ async def test_reseed_warms_cache_even_on_zero_db_spend():
finally:
ps.spend_counter_cache = orig_counter
ps.prisma_client = orig_prisma
# ---------------------------------------------------------------------------
# Lazy feature loading (LazyFeatureMiddleware) — verifies that optional
# routers are NOT imported at module load and ARE imported on first request
# to a matching path prefix. The same module isn't re-imported on subsequent
# requests.
# ---------------------------------------------------------------------------
import sys
class TestLazyFeatureRegistry:
"""Sanity checks on the registry shape — guards against accidental edits."""
def test_registry_entries_have_required_fields(self):
from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature
assert len(LAZY_FEATURES) > 0
for feat in LAZY_FEATURES:
assert isinstance(feat, LazyFeature)
assert feat.name
assert feat.module_path
assert feat.path_prefixes
assert all(p.startswith("/") for p in feat.path_prefixes)
assert callable(feat.register_fn)
def test_registry_names_unique(self):
from litellm.proxy._lazy_features import LAZY_FEATURES
names = [f.name for f in LAZY_FEATURES]
assert len(names) == len(set(names)), "duplicate feature names"
class TestLazyFeaturesNotImportedAtStartup:
"""
The whole point of the refactor: gated feature modules must NOT be
present in `sys.modules` immediately after `proxy_server` imports.
"""
def test_heavy_modules_absent_at_startup(self):
# Force a fresh `proxy_server` import in a subprocess so other tests
# in this run (which may have triggered lazy loads via the TestClient)
# don't pollute the result.
import subprocess
check = (
"import sys; "
"from litellm.proxy.proxy_server import app; " # noqa: F401
"heavy = ["
"'litellm.proxy._experimental.mcp_server.rest_endpoints',"
"'litellm.proxy._experimental.mcp_server.server',"
"'litellm.proxy.management_endpoints.config_override_endpoints',"
"'litellm.proxy.guardrails.guardrail_endpoints',"
"'litellm.proxy.openai_evals_endpoints.endpoints',"
"]; "
"still_present = [m for m in heavy if m in sys.modules]; "
"print('PRESENT_AT_STARTUP:', still_present)"
)
result = subprocess.run(
[sys.executable, "-c", check],
capture_output=True,
text=True,
timeout=120,
)
# Last non-empty line of stdout (skip warnings printed before)
out_lines = [
line for line in result.stdout.strip().splitlines() if line.strip()
]
report = next((line for line in out_lines if "PRESENT_AT_STARTUP" in line), "")
assert report, f"no report emitted (stderr: {result.stderr[-500:]})"
assert (
"PRESENT_AT_STARTUP: []" in report
), f"expected no heavy modules at startup, got: {report}"
class TestLazyFeatureMiddleware:
"""Behavior of the middleware itself, exercised in isolation."""
@pytest.mark.asyncio
async def test_first_request_triggers_load_subsequent_does_not(self):
from fastapi import FastAPI
from litellm.proxy._lazy_features import (
LazyFeature,
LazyFeatureMiddleware,
)
loads = []
def fake_register(app, module):
loads.append(getattr(module, "__name__", "?"))
feat = LazyFeature(
name="dummy",
module_path="json", # any always-importable stdlib module
path_prefixes=("/dummy",),
register_fn=fake_register,
)
# Build a minimal ASGI receiver to satisfy the middleware contract
async def downstream(scope, receive, send):
# echo back; no-op handler
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b""})
target_app = FastAPI()
mw = LazyFeatureMiddleware(downstream, fastapi_app=target_app, features=(feat,))
async def receive():
return {"type": "http.request", "body": b"", "more_body": False}
sent: list = []
async def send(message):
sent.append(message)
# First request matching the prefix triggers register
await mw(
{"type": "http", "path": "/dummy/x", "method": "GET", "headers": []},
receive,
send,
)
assert loads == ["json"]
# Second matching request must NOT re-register
sent.clear()
await mw(
{"type": "http", "path": "/dummy/y", "method": "GET", "headers": []},
receive,
send,
)
assert loads == ["json"], "register_fn called twice for the same feature"
# Non-matching path must not trigger anything
await mw(
{"type": "http", "path": "/unrelated", "method": "GET", "headers": []},
receive,
send,
)
assert loads == ["json"]
@pytest.mark.asyncio
async def test_concurrent_first_requests_only_register_once(self):
"""
Two requests to the same prefix arriving in parallel must result in
exactly one `register_fn` invocation — the lock prevents the import +
register from racing with itself.
"""
from fastapi import FastAPI
from litellm.proxy._lazy_features import (
LazyFeature,
LazyFeatureMiddleware,
)
loads = []
def slow_register(app, module):
loads.append(getattr(module, "__name__", "?"))
feat = LazyFeature(
name="dummy_concurrent",
module_path="json",
path_prefixes=("/dummy_c",),
register_fn=slow_register,
)
async def downstream(scope, receive, send):
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b""})
target_app = FastAPI()
mw = LazyFeatureMiddleware(downstream, fastapi_app=target_app, features=(feat,))
async def receive():
return {"type": "http.request", "body": b"", "more_body": False}
sent: list = []
async def send(message):
sent.append(message)
async def hit():
await mw(
{
"type": "http",
"path": "/dummy_c/x",
"method": "GET",
"headers": [],
},
receive,
send,
)
await asyncio.gather(hit(), hit(), hit(), hit(), hit())
assert loads == [
"json"
], f"expected one registration despite concurrent first hits, got {loads}"
@pytest.mark.asyncio
async def test_failing_import_does_not_loop(self):
"""
If a feature's module can't be imported, the middleware should mark it
loaded anyway so subsequent requests don't repeatedly retry the failing
import (which would amplify the cost on every request).
"""
from fastapi import FastAPI
from litellm.proxy._lazy_features import (
LazyFeature,
LazyFeatureMiddleware,
)
attempts = []
def fail_register(app, module):
attempts.append("called")
raise RuntimeError("boom")
feat = LazyFeature(
name="failing",
module_path="json",
path_prefixes=("/fail",),
register_fn=fail_register,
)
async def downstream(scope, receive, send):
await send({"type": "http.response.start", "status": 200, "headers": []})
await send({"type": "http.response.body", "body": b""})
target_app = FastAPI()
mw = LazyFeatureMiddleware(downstream, fastapi_app=target_app, features=(feat,))
async def receive():
return {"type": "http.request", "body": b"", "more_body": False}
sent: list = []
async def send(message):
sent.append(message)
for _ in range(3):
await mw(
{"type": "http", "path": "/fail/x", "method": "GET", "headers": []},
receive,
send,
)
assert attempts == [
"called"
], f"failing register_fn should be invoked once, not on every request; got {attempts}"

15
tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py
View file

@ -786,8 +786,23 @@ class TestVectorStoreManagementEndpointsExist:
- POST /vector_store/info
- POST /vector_store/update
"""
import importlib
from litellm.proxy._lazy_features import LAZY_FEATURES
from litellm.proxy.proxy_server import app
# Force-register the lazy vector_store_management routes so the
# assertions can find them.
already_registered = any(
getattr(r, "path", None) == "/vector_store/new" for r in app.routes
)
if not already_registered:
for feat in LAZY_FEATURES:
if feat.name == "vector_store_management":
module = importlib.import_module(feat.module_path)
feat.register_fn(app, module)
break
# Define expected endpoints
expected_endpoints = [
("POST", "/vector_store/new"),