+ Complete CLI Login
+ Enter the verification code shown in your terminal to finish this login.
+
+
+
+
+ """
+
+
+@router.post("/sso/cli/start", tags=["experimental"], include_in_schema=False)
+async def cli_sso_start(request: Request):
+ from litellm.proxy.proxy_server import general_settings, user_api_key_cache
+
+ _check_cli_sso_start_rate_limit(
+ request=request,
+ cache=user_api_key_cache,
+ use_x_forwarded_for=bool(
+ (general_settings or {}).get("use_x_forwarded_for", False)
+ ),
+ )
+
+ login_id = f"cli-{secrets.token_urlsafe(24)}"
+ poll_secret = secrets.token_urlsafe(32)
+ user_code = _generate_cli_sso_user_code()
+
+ flow = {
+ "poll_secret_hash": _hash_cli_sso_secret(poll_secret),
+ "user_code_hash": _hash_cli_sso_secret(_normalize_cli_sso_user_code(user_code)),
+ "sso_complete": False,
+ "user_code_verified": False,
+ "session_data": None,
+ }
+ _set_cli_sso_flow(login_id=login_id, cache=user_api_key_cache, flow=flow)
+
+ return {
+ "login_id": login_id,
+ "poll_secret": poll_secret,
+ "user_code": user_code,
+ "expires_in": CLI_SSO_SESSION_TTL_SECONDS,
+ }
+
+
+@router.post(
+ "/sso/cli/complete/{login_id}", tags=["experimental"], include_in_schema=False
+)
+async def cli_sso_complete(request: Request, login_id: str):
+ from fastapi.responses import HTMLResponse
+
+ from litellm.proxy.common_utils.html_forms.cli_sso_success import (
+ render_cli_sso_success_page,
+ )
+ from litellm.proxy.proxy_server import user_api_key_cache
+
+ flow = _get_cli_sso_flow_or_raise(login_id=login_id, cache=user_api_key_cache)
+ if not flow.get("sso_complete") or not flow.get("session_data"):
+ raise HTTPException(status_code=400, detail="CLI login is not ready")
+
+ body = (await request.body()).decode("utf-8")
+ form_values = parse_qs(body)
+ supplied_user_code = (form_values.get("user_code") or [""])[0]
+ supplied_browser_complete_token = (
+ form_values.get("browser_complete_token") or [""]
+ )[0]
+ supplied_user_code_hash = _hash_cli_sso_secret(
+ _normalize_cli_sso_user_code(supplied_user_code)
+ )
+ supplied_browser_complete_token_hash = _hash_cli_sso_secret(
+ supplied_browser_complete_token
+ )
+
+ expected_user_code_hash = flow.get("user_code_hash")
+ if not isinstance(expected_user_code_hash, str) or not secrets.compare_digest(
+ supplied_user_code_hash, expected_user_code_hash
+ ):
+ raise HTTPException(status_code=400, detail="Invalid verification code")
+
+ expected_browser_complete_token_hash = flow.get("browser_complete_token_hash")
+ if not isinstance(
+ expected_browser_complete_token_hash, str
+ ) or not secrets.compare_digest(
+ supplied_browser_complete_token_hash, expected_browser_complete_token_hash
+ ):
+ raise HTTPException(status_code=400, detail="Invalid verification code")
+
+ flow["user_code_verified"] = True
+ _set_cli_sso_flow(login_id=login_id, cache=user_api_key_cache, flow=flow)
+
+ html_content = render_cli_sso_success_page()
+ return HTMLResponse(content=html_content, status_code=200)
def normalize_email(email: Optional[str]) -> Optional[str]:
@@ -333,6 +586,7 @@ async def google_login(
from litellm.proxy.proxy_server import (
premium_user,
prisma_client,
+ user_api_key_cache,
user_custom_ui_sso_sign_in_handler,
)
@@ -382,14 +636,15 @@ async def google_login(
redirect_url = SSOAuthenticationHandler.get_redirect_url_for_sso(
request=request,
sso_callback_route="sso/callback",
- existing_key=existing_key,
)
- # Store CLI key in state for OAuth flow
+ if source == LITELLM_CLI_SOURCE_IDENTIFIER:
+ _get_cli_sso_flow_or_raise(login_id=key, cache=user_api_key_cache)
+
+ # Store CLI login handle in state for OAuth flow
cli_state: Optional[str] = SSOAuthenticationHandler._get_cli_state(
source=source,
key=key,
- existing_key=existing_key,
)
# check if user defined a custom auth sso sign in handler, if yes, use it
@@ -423,6 +678,7 @@ async def google_login(
google_client_id=google_client_id,
generic_client_id=generic_client_id,
state=cli_state,
+ request=request,
)
if return_to is not None and sso_redirect is not None:
if SSOAuthenticationHandler._validate_return_to(return_to):
@@ -904,6 +1160,30 @@ async def get_generic_sso_response(
authorization_code = request.query_params.get("code")
if code_verifier:
+ # State-to-session-cookie binding. The non-PKCE branch below
+ # delegates to fastapi-sso's ``verify_and_process``, which
+ # performs its own session-cookie check. The PKCE branch
+ # bypasses that helper, so we validate the URL ``state``
+ # against the ``litellm_oauth_state`` cookie set on the
+ # redirect response — without this an attacker can pre-mint
+ # a state + cached PKCE verifier and hijack a victim's auth
+ # code (Login-CSRF / token theft).
+ url_state = request.query_params.get("state")
+ cookie_state = request.cookies.get("litellm_oauth_state")
+ if (
+ not url_state
+ or not cookie_state
+ or not secrets.compare_digest(url_state, cookie_state)
+ ):
+ raise ProxyException(
+ message=(
+ "Invalid OAuth state parameter — does not match "
+ "the browser-bound state cookie."
+ ),
+ type=ProxyErrorTypes.auth_error,
+ param="state",
+ code=status.HTTP_400_BAD_REQUEST,
+ )
if not authorization_code:
raise ProxyException(
message="Missing authorization code in callback",
@@ -1050,7 +1330,7 @@ async def get_existing_user_info_from_db(
user_id: Optional[str],
user_email: Optional[str],
prisma_client: PrismaClient,
- user_api_key_cache: DualCache,
+ user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging,
) -> Optional[LiteLLM_UserTable]:
try:
@@ -1074,7 +1354,7 @@ async def get_existing_user_info_from_db(
async def get_user_info_from_db(
result: Union[CustomOpenID, OpenID, dict],
prisma_client: PrismaClient,
- user_api_key_cache: DualCache,
+ user_api_key_cache: UserApiKeyCache,
proxy_logging_obj: ProxyLogging,
user_email: Optional[str],
user_defined_values: Optional[SSOUserDefinedValues],
@@ -1194,7 +1474,7 @@ async def _sync_user_role_from_jwt_role_map(
received_response: Optional[dict],
user_info: Optional[Union[LiteLLM_UserTable, NewUserResponse]],
prisma_client: PrismaClient,
- user_api_key_cache: DualCache,
+ user_api_key_cache: UserApiKeyCache,
user_defined_values: Optional[SSOUserDefinedValues],
) -> None:
"""
@@ -1233,11 +1513,8 @@ async def _sync_user_role_from_jwt_role_map(
user_info.user_role = mapped_role.value
await user_api_key_cache.async_set_cache(
key=user_info.user_id,
- value=(
- user_info.model_dump()
- if hasattr(user_info, "model_dump")
- else dict(user_info)
- ),
+ value=user_info,
+ model_type=LiteLLM_UserTable,
)
@@ -1392,18 +1669,12 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa:
)
if state and state.startswith(f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:"):
- # Extract the key ID and existing_key from the state
- # State format: {PREFIX}:{key}:{existing_key} or {PREFIX}:{key}
- state_parts = state.split(":", 2) # Split into max 3 parts
+ # State format: {PREFIX}:{login_id}
+ state_parts = state.split(":", 1)
key_id = state_parts[1] if len(state_parts) > 1 else None
- existing_key = state_parts[2] if len(state_parts) > 2 else None
- verbose_proxy_logger.info(
- f"CLI SSO callback detected for key: {key_id}, existing_key: {existing_key}"
- )
- return await cli_sso_callback(
- request=request, key=key_id, existing_key=existing_key, result=result
- )
+ verbose_proxy_logger.info("CLI SSO callback detected")
+ return await cli_sso_callback(request=request, key=key_id, result=result)
# Control-plane cross-origin: read return_to from cookie.
# Starlette's cookie_parser already handles RFC 2109 unquoting.
@@ -1424,13 +1695,10 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa:
async def cli_sso_callback(
request: Request,
key: Optional[str] = None,
- existing_key: Optional[str] = None,
result: Optional[Union[OpenID, dict]] = None,
):
"""CLI SSO callback - stores session info for JWT generation on polling"""
- verbose_proxy_logger.info(
- f"CLI SSO callback for key: {key}, existing_key: {existing_key}"
- )
+ verbose_proxy_logger.info("CLI SSO callback")
from litellm.proxy.proxy_server import (
prisma_client,
@@ -1438,11 +1706,7 @@ async def cli_sso_callback(
user_api_key_cache,
)
- if not key or not key.startswith("sk-"):
- raise HTTPException(
- status_code=400,
- detail="Invalid key parameter. Must be a valid key ID starting with 'sk-'",
- )
+ flow = _get_cli_sso_flow_or_raise(login_id=key, cache=user_api_key_cache)
if prisma_client is None:
raise HTTPException(
@@ -1480,9 +1744,6 @@ async def cli_sso_callback(
status_code=500, detail="Failed to retrieve user information from SSO"
)
- # Store session info in cache (10 min TTL)
- from litellm.constants import CLI_SSO_SESSION_CACHE_KEY_PREFIX
-
# Get all teams from user_info - CLI will let user select which one
teams: List[str] = []
if hasattr(user_info, "teams") and user_info.teams:
@@ -1523,21 +1784,25 @@ async def cli_sso_callback(
"team_details": team_details,
}
- cache_key = f"{CLI_SSO_SESSION_CACHE_KEY_PREFIX}:{key}"
- user_api_key_cache.set_cache(key=cache_key, value=session_data, ttl=600)
+ flow["session_data"] = session_data
+ flow["sso_complete"] = True
+ browser_complete_token = secrets.token_urlsafe(32)
+ flow["browser_complete_token_hash"] = _hash_cli_sso_secret(
+ browser_complete_token
+ )
+ _set_cli_sso_flow(login_id=cast(str, key), cache=user_api_key_cache, flow=flow)
verbose_proxy_logger.info(
f"Stored CLI SSO session for user: {user_info.user_id}, teams: {teams}, num_teams: {len(teams)}"
)
- # Return success page
from fastapi.responses import HTMLResponse
- from litellm.proxy.common_utils.html_forms.cli_sso_success import (
- render_cli_sso_success_page,
+ verify_url = str(request.url_for("cli_sso_complete", login_id=key))
+ html_content = _render_cli_sso_verification_page(
+ verify_url=verify_url,
+ browser_complete_token=browser_complete_token,
)
-
- html_content = render_cli_sso_success_page()
return HTMLResponse(content=html_content, status_code=200)
except Exception as e:
@@ -1548,7 +1813,11 @@ async def cli_sso_callback(
@router.get("/sso/cli/poll/{key_id}", tags=["experimental"], include_in_schema=False)
-async def cli_poll_key(key_id: str, team_id: Optional[str] = None):
+async def cli_poll_key(
+ key_id: str,
+ team_id: Optional[str] = None,
+ x_litellm_cli_poll_secret: Optional[str] = Header(default=None),
+):
"""
CLI polling endpoint - retrieves session from cache and generates JWT.
@@ -1557,22 +1826,25 @@ async def cli_poll_key(key_id: str, team_id: Optional[str] = None):
2. Second poll (with team_id): Generates JWT with selected team and deletes session
Args:
- key_id: The session key ID
+ key_id: The CLI login session ID
team_id: Optional team ID to assign to the JWT. If provided, must be one of user's teams.
"""
- from litellm.constants import CLI_SSO_SESSION_CACHE_KEY_PREFIX
from litellm.proxy.auth.auth_checks import ExperimentalUIJWTToken
from litellm.proxy.proxy_server import user_api_key_cache
- if not key_id.startswith("sk-"):
- raise HTTPException(status_code=400, detail="Invalid key ID format")
-
try:
- # Look up session in cache
- cache_key = f"{CLI_SSO_SESSION_CACHE_KEY_PREFIX}:{key_id}"
- session_data = user_api_key_cache.get_cache(key=cache_key)
+ flow = _get_cli_sso_flow_or_raise(login_id=key_id, cache=user_api_key_cache)
+ if not _verify_cli_sso_poll_secret(
+ flow=flow, poll_secret=x_litellm_cli_poll_secret
+ ):
+ raise HTTPException(status_code=403, detail="Invalid CLI polling secret")
- if session_data:
+ if not flow.get("sso_complete") or not flow.get("user_code_verified"):
+ return {"status": "pending"}
+
+ session_data = flow.get("session_data")
+
+ if isinstance(session_data, dict):
user_teams = session_data.get("teams", [])
user_team_details = session_data.get("team_details")
user_id = session_data["user_id"]
@@ -1632,7 +1904,7 @@ async def cli_poll_key(key_id: str, team_id: Optional[str] = None):
)
# Delete cache entry (single-use)
- user_api_key_cache.delete_cache(key=cache_key)
+ user_api_key_cache.delete_cache(key=_get_cli_sso_flow_cache_key(key_id))
verbose_proxy_logger.info(
f"CLI JWT generated for user: {user_id}, team: {team_id}"
@@ -1650,6 +1922,8 @@ async def cli_poll_key(key_id: str, team_id: Optional[str] = None):
else:
return {"status": "pending"}
+ except HTTPException:
+ raise
except Exception as e:
verbose_proxy_logger.error(f"Error polling for CLI JWT: {e}")
raise HTTPException(
@@ -1898,6 +2172,7 @@ class SSOAuthenticationHandler:
microsoft_client_id: Optional[str] = None,
generic_client_id: Optional[str] = None,
state: Optional[str] = None,
+ request: Optional[Request] = None,
) -> Optional[RedirectResponse]:
"""
Step 1. Call Get Login Redirect for the SSO provider. Send the redirect response to `redirect_url`
@@ -1907,6 +2182,8 @@ class SSOAuthenticationHandler:
google_client_id (Optional[str], optional): The Google Client ID. Defaults to None.
microsoft_client_id (Optional[str], optional): The Microsoft Client ID. Defaults to None.
generic_client_id (Optional[str], optional): The Generic Client ID. Defaults to None.
+ request: Optional FastAPI request, used to drive the ``Secure``
+ attribute on the ``litellm_oauth_state`` CSRF cookie.
Returns:
RedirectResponse: The redirect response from the SSO provider.
@@ -2017,6 +2294,7 @@ class SSOAuthenticationHandler:
generic_sso=generic_sso,
state=state,
generic_authorization_endpoint=generic_authorization_endpoint,
+ request=request,
)
raise ValueError(
"Unknown SSO provider. Please setup SSO with client IDs https://docs.litellm.ai/docs/proxy/admin_ui_sso"
@@ -2027,6 +2305,7 @@ class SSOAuthenticationHandler:
generic_sso: Any,
state: Optional[str] = None,
generic_authorization_endpoint: Optional[str] = None,
+ request: Optional[Request] = None,
) -> Optional[RedirectResponse]:
"""
Get the redirect response for Generic SSO
@@ -2036,10 +2315,13 @@ class SSOAuthenticationHandler:
from litellm.proxy.proxy_server import redis_usage_cache, user_api_key_cache
with generic_sso:
- # TODO: state should be a random string and added to the user session with cookie
- # or a cryptographicly signed state that we can verify stateless
- # For simplification we are using a static state, this is not perfect but some
- # SSO providers do not allow stateless verification
+ # State is bound to the caller's browser via a ``litellm_oauth_state``
+ # HttpOnly cookie set on the redirect response below; the SSO
+ # callback validates the URL ``state`` against that cookie before
+ # completing the PKCE token exchange. Without this binding, an
+ # attacker who pre-mints a state + a cached PKCE verifier can hand
+ # the link to a victim and capture the resulting access token
+ # (Login CSRF / token theft).
(
redirect_params,
code_verifier,
@@ -2106,6 +2388,31 @@ class SSOAuthenticationHandler:
# Update the redirect response
redirect_response.headers["location"] = new_url
+
+ # Bind state to the user's browser session. The /callback
+ # handler validates the URL ``state`` against this cookie via
+ # ``secrets.compare_digest`` before exchanging the PKCE
+ # code_verifier. Only set the cookie when PKCE is in use
+ # (i.e. inside this ``code_verifier`` branch) so two
+ # concurrent SSO sessions — one PKCE, one plain — cannot
+ # overwrite each other's state cookie.
+ state_value = redirect_params.get("state")
+ if state_value and redirect_response is not None:
+ # Production-safe default: require HTTPS for the
+ # CSRF-protection cookie unless we can prove the
+ # incoming request is HTTP (local dev). Without
+ # ``Secure`` the cookie is sent over plain HTTP,
+ # letting a network observer read and replay the
+ # state value and bypass this protection.
+ secure_flag = request is None or request.url.scheme == "https"
+ redirect_response.set_cookie(
+ key="litellm_oauth_state",
+ value=state_value,
+ max_age=600,
+ httponly=True,
+ samesite="lax",
+ secure=secure_flag,
+ )
return redirect_response
@staticmethod
@@ -2393,20 +2700,15 @@ class SSOAuthenticationHandler:
This is used to authenticate through the CLI login flow.
- The state parameter format is: {PREFIX}:{key}:{existing_key}
- - If existing_key is provided, it's included in the state
+ The state parameter format is: {PREFIX}:{login_id}
- The state parameter is used to pass data through the OAuth flow without changing the callback URL
"""
from litellm.constants import (
LITELLM_CLI_SESSION_TOKEN_PREFIX,
- LITELLM_CLI_SOURCE_IDENTIFIER,
)
if source == LITELLM_CLI_SOURCE_IDENTIFIER and key:
- if existing_key:
- return f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:{key}:{existing_key}"
- else:
- return f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:{key}"
+ return f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:{key}"
else:
return None
@@ -3728,6 +4030,7 @@ async def debug_sso_login(request: Request):
microsoft_client_id=microsoft_client_id,
google_client_id=google_client_id,
generic_client_id=generic_client_id,
+ request=request,
)
diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py
index 4de29e04092..a50ce1d3c48 100644
--- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py
+++ b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py
@@ -440,6 +440,15 @@ def _resolve_fetch_kwargs(
kwargs: Dict[str, Any] = {"start_date": start_date, "end_date": end_date}
if fn_name == "get_usage_data":
if not is_admin:
+ if user_id is None:
+ # Defense-in-depth: the endpoint guard in usage_endpoints/endpoints.py
+ # should have already rejected this. If we ever reach here it means
+ # a future caller invoked the helper without scoping — fail loudly
+ # rather than issuing an unfiltered global query.
+ raise ValueError(
+ "Non-admin caller has user_id=None; refusing to issue an "
+ "unscoped query. Endpoint-level guard missing."
+ )
kwargs["user_id"] = user_id
elif fn_args.get("user_id"):
kwargs["user_id"] = fn_args["user_id"]
diff --git a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py
index 0dbe518afb7..d0df80fed0d 100644
--- a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py
+++ b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py
@@ -44,13 +44,17 @@ async def usage_ai_chat(
"""
from litellm.proxy.management_endpoints.common_utils import (
_user_has_admin_view,
+ require_caller_user_id_for_non_admin,
)
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
stream_usage_ai_chat,
)
is_admin = _user_has_admin_view(user_api_key_dict)
- user_id = user_api_key_dict.user_id
+ if is_admin:
+ user_id = user_api_key_dict.user_id
+ else:
+ user_id = require_caller_user_id_for_non_admin(user_api_key_dict)
messages = [{"role": m.role, "content": m.content} for m in data.messages]
return StreamingResponse(
diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py
index 7599e11bdef..d3b225e6e4d 100644
--- a/litellm/proxy/management_helpers/audit_logs.py
+++ b/litellm/proxy/management_helpers/audit_logs.py
@@ -21,6 +21,28 @@ from litellm.proxy._types import (
from litellm.types.utils import StandardAuditLogPayload
_audit_log_callback_cache: Dict[str, CustomLogger] = {}
+ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY = "allow_litellm_changed_by_header"
+
+
+def _allows_litellm_changed_by_header(user_api_key_dict: UserAPIKeyAuth) -> bool:
+ for admin_metadata in (user_api_key_dict.metadata, user_api_key_dict.team_metadata):
+ if (
+ isinstance(admin_metadata, dict)
+ and admin_metadata.get(ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY) is True
+ ):
+ return True
+ return False
+
+
+def get_audit_log_changed_by(
+ *,
+ litellm_changed_by: Optional[str],
+ user_api_key_dict: UserAPIKeyAuth,
+ litellm_proxy_admin_name: Optional[str],
+) -> Optional[str]:
+ if litellm_changed_by and _allows_litellm_changed_by_header(user_api_key_dict):
+ return litellm_changed_by
+ return user_api_key_dict.user_id or litellm_proxy_admin_name
def _resolve_audit_log_callback(name: str) -> Optional[CustomLogger]:
@@ -143,8 +165,10 @@ async def create_object_audit_log(
if _store_audit_logs is not True:
return
- _changed_by = (
- litellm_changed_by or user_api_key_dict.user_id or litellm_proxy_admin_name
+ _changed_by = get_audit_log_changed_by(
+ litellm_changed_by=litellm_changed_by,
+ user_api_key_dict=user_api_key_dict,
+ litellm_proxy_admin_name=litellm_proxy_admin_name,
)
await create_audit_log_for_update(
diff --git a/litellm/proxy/management_helpers/team_member_permission_checks.py b/litellm/proxy/management_helpers/team_member_permission_checks.py
index e035168ca00..50339210a6e 100644
--- a/litellm/proxy/management_helpers/team_member_permission_checks.py
+++ b/litellm/proxy/management_helpers/team_member_permission_checks.py
@@ -1,6 +1,5 @@
from typing import List, Optional
-from litellm.caching import DualCache
from litellm.proxy._types import (
KeyManagementRoutes,
LiteLLM_TeamTableCachedObj,
@@ -12,6 +11,7 @@ from litellm.proxy._types import (
ProxyException,
UserAPIKeyAuth,
)
+from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.auth.auth_checks import get_team_object
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.utils import PrismaClient
@@ -65,7 +65,7 @@ class TeamMemberPermissionChecks:
user_api_key_dict: UserAPIKeyAuth,
route: KeyManagementRoutes,
prisma_client: PrismaClient,
- user_api_key_cache: DualCache,
+ user_api_key_cache: UserApiKeyCache,
existing_key_row: LiteLLM_VerificationToken,
):
"""
diff --git a/litellm/proxy/middleware/prometheus_auth_middleware.py b/litellm/proxy/middleware/prometheus_auth_middleware.py
index 6bdff59da52..3b30fd3d63c 100644
--- a/litellm/proxy/middleware/prometheus_auth_middleware.py
+++ b/litellm/proxy/middleware/prometheus_auth_middleware.py
@@ -3,6 +3,7 @@ Prometheus Auth Middleware - Pure ASGI implementation
"""
import json
+from typing import Any, List, MutableMapping
from fastapi import Request
from starlette.types import ASGIApp, Receive, Scope, Send
@@ -40,8 +41,17 @@ class PrometheusAuthMiddleware:
# Only run auth if configured to do so
if litellm.require_auth_for_metrics_endpoint is True:
- # Construct Request only when auth is actually needed
- request = Request(scope, receive)
+ # user_api_key_auth reads the request body, which consumes ASGI `receive`.
+ # Buffer those messages and replay them for the inner app; otherwise a
+ # successful auth would forward an exhausted receive and /metrics hangs.
+ buffered_messages: List[MutableMapping[str, Any]] = []
+
+ async def receive_for_auth() -> MutableMapping[str, Any]:
+ message = await receive()
+ buffered_messages.append(message)
+ return message
+
+ request = Request(scope, receive_for_auth)
api_key = request.headers.get(_AUTHORIZATION_HEADER) or ""
try:
@@ -70,5 +80,18 @@ class PrometheusAuthMiddleware:
)
return
+ replay_idx = 0
+
+ async def receive_replay() -> MutableMapping[str, Any]:
+ nonlocal replay_idx
+ if replay_idx < len(buffered_messages):
+ msg = buffered_messages[replay_idx]
+ replay_idx += 1
+ return msg
+ return await receive()
+
+ await self.app(scope, receive_replay, send)
+ return
+
# Pass through to the inner application
await self.app(scope, receive, send)
diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
index 6521abffb85..ce103f806e1 100644
--- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
@@ -47,6 +47,8 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
)
from litellm.proxy.utils import is_known_model
from litellm.proxy.vector_store_endpoints.utils import (
+ assert_user_can_access_vector_store,
+ get_litellm_managed_vector_store,
is_allowed_to_call_vector_store_endpoint,
)
from litellm.secret_managers.main import get_secret_str
@@ -533,6 +535,10 @@ async def milvus_proxy_route(
)
if vector_store is None:
raise Exception(f"Vector store not found for {vector_store_name}")
+ await assert_user_can_access_vector_store(
+ vector_store=vector_store,
+ user_api_key_dict=user_api_key_dict,
+ )
litellm_params = vector_store.get("litellm_params") or {}
auth_credentials = provider_config.get_auth_credentials(
litellm_params=litellm_params
@@ -1438,6 +1444,10 @@ async def azure_proxy_route(
)
if vector_store is None:
raise Exception(f"Vector store not found for {vector_store_name}")
+ await assert_user_can_access_vector_store(
+ vector_store=vector_store,
+ user_api_key_dict=user_api_key_dict,
+ )
litellm_params = vector_store.get("litellm_params") or {}
auth_credentials = provider_config.get_auth_credentials(
litellm_params=litellm_params
@@ -1777,6 +1787,11 @@ async def _base_vertex_proxy_route(
request=request,
api_key=api_key_to_use,
)
+ if router_credentials is not None:
+ await assert_user_can_access_vector_store(
+ vector_store=router_credentials,
+ user_api_key_dict=user_api_key_dict,
+ )
vertex_project: Optional[str] = get_vertex_project_id_from_url(endpoint)
vertex_location: Optional[str] = get_vertex_location_from_url(endpoint)
@@ -1913,11 +1928,11 @@ async def vertex_discovery_proxy_route(
"Extracted vector store ID from endpoint: %s", vector_store_id
)
- # Retrieve vector store credentials from the registry
- vector_store_credentials = (
- passthrough_endpoint_router.get_vector_store_credentials(
- vector_store_id=vector_store_id
- )
+ # Retrieve LiteLLM-managed vector store credentials if the datastore id
+ # is registered with LiteLLM. Unknown datastore ids keep the existing
+ # direct Vertex pass-through behavior.
+ vector_store_credentials = await get_litellm_managed_vector_store(
+ vector_store_id=vector_store_id
)
if vector_store_credentials:
@@ -1925,7 +1940,7 @@ async def vertex_discovery_proxy_route(
"Found vector store credentials for ID: %s", vector_store_id
)
else:
- verbose_proxy_logger.warning(
+ verbose_proxy_logger.debug(
"Vector store ID %s found in endpoint but no credentials found in registry",
vector_store_id,
)
diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py
index a8c5562d4d6..6277f6b4a75 100644
--- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py
+++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/assembly_passthrough_logging_handler.py
@@ -1,6 +1,7 @@
import asyncio
import json
import time
+import urllib.parse
from datetime import datetime
from typing import Literal, Optional
from urllib.parse import urlparse
@@ -203,8 +204,16 @@ class AssemblyAIPassthroughLoggingHandler:
)
if _api_key is None:
raise ValueError("AssemblyAI API key not found")
+ if (
+ any(c in transcript_id for c in ("/", "\\", "#", "?"))
+ or ".." in transcript_id
+ ):
+ raise ValueError(
+ f"Invalid transcript_id {transcript_id!r}: contains disallowed characters"
+ )
+ safe_transcript_id = urllib.parse.quote(transcript_id, safe="")
try:
- url = f"{_base_url}/v2/transcript/{transcript_id}"
+ url = f"{_base_url}/v2/transcript/{safe_transcript_id}"
headers = {
"Authorization": f"Bearer {_api_key}",
"Content-Type": "application/json",
diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
index 77eb3a5ee0c..cc6c26fdf90 100644
--- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
@@ -41,7 +41,6 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.passthrough import BasePassthroughUtils
from litellm.proxy._types import (
- CommonProxyErrors,
ConfigFieldInfo,
ConfigFieldUpdate,
LiteLLMRoutes,
@@ -2325,12 +2324,10 @@ async def _register_pass_through_endpoint(
dependencies = None
if auth is not None and str(auth).lower() == "true":
- if premium_user is not True:
- raise ValueError(
- "Error Setting Authentication on Pass Through Endpoint: {}".format(
- CommonProxyErrors.not_premium_user.value
- )
- )
+ # Authentication on a pass-through endpoint used to be enterprise-only.
+ # That left OSS with no safe configuration: auth=True raised at startup
+ # unless the operator had a license. The safe option must always be free,
+ # and unauthenticated forwarding should require explicit opt-in.
dependencies = [Depends(user_api_key_auth)]
if path not in LiteLLMRoutes.openai_routes.value:
LiteLLMRoutes.openai_routes.value.append(path)
diff --git a/litellm/proxy/policy_engine/attachment_registry.py b/litellm/proxy/policy_engine/attachment_registry.py
index 6d1096d5ee9..8d5d8116919 100644
--- a/litellm/proxy/policy_engine/attachment_registry.py
+++ b/litellm/proxy/policy_engine/attachment_registry.py
@@ -220,6 +220,7 @@ class AttachmentRegistry:
attachment: PolicyAttachment object to add
"""
self._attachments.append(attachment)
+ self._initialized = True
verbose_proxy_logger.debug(f"Added attachment for policy: {attachment.policy}")
def remove_attachments_for_policy(self, policy_name: str) -> int:
diff --git a/litellm/proxy/policy_engine/policy_registry.py b/litellm/proxy/policy_engine/policy_registry.py
index d3df16afde6..75017c46603 100644
--- a/litellm/proxy/policy_engine/policy_registry.py
+++ b/litellm/proxy/policy_engine/policy_registry.py
@@ -226,6 +226,7 @@ class PolicyRegistry:
policy: Policy object to add
"""
self._policies[policy_name] = policy
+ self._initialized = True
verbose_proxy_logger.debug(f"Added/updated policy: {policy_name}")
def remove_policy(self, policy_name: str) -> bool:
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 12229955299..3db13e8ada6 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -6,6 +6,7 @@ import inspect
import io
import os
import random
+import re
import secrets
import shutil
import subprocess
@@ -78,8 +79,11 @@ from litellm.proxy._types import (
InvitationNew,
InvitationUpdate,
Litellm_EntityType,
+ LiteLLM_EndUserTable,
LiteLLM_JWTAuth,
+ LiteLLM_TagTable,
LiteLLM_TeamTable,
+ LiteLLM_TeamTableCachedObj,
LiteLLM_UserTable,
LitellmUserRoles,
PassThroughGenericEndpoint,
@@ -91,8 +95,10 @@ from litellm.proxy._types import (
TeamDefaultSettings,
TokenCountRequest,
TransformRequestBody,
+ UI_TEAM_ID,
UserAPIKeyAuth,
)
+from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
from litellm.proxy.common_utils.callback_utils import (
normalize_callback_names,
process_callback,
@@ -205,6 +211,7 @@ from litellm import Router
from litellm._logging import verbose_proxy_logger, verbose_router_logger
from litellm.caching.caching import DualCache, RedisCache
from litellm.caching.redis_cluster_cache import RedisClusterCache
+from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.constants import (
_REALTIME_BODY_CACHE_SIZE,
APSCHEDULER_COALESCE,
@@ -235,37 +242,11 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
-from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import (
- router as mcp_byok_oauth_router,
-)
-from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
- router as mcp_discoverable_endpoints_router,
-)
-from litellm.proxy._experimental.mcp_server.rest_endpoints import (
- router as mcp_rest_endpoints_router,
-)
-from litellm.proxy._experimental.mcp_server.server import app as mcp_app
-from litellm.proxy._experimental.mcp_server.tool_registry import (
- global_mcp_tool_registry,
-)
from litellm.proxy._types import *
-from litellm.proxy.agent_endpoints.a2a_endpoints import router as a2a_router
-from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
-from litellm.proxy.agent_endpoints.endpoints import router as agent_endpoints_router
-from litellm.proxy.agent_endpoints.model_list_helpers import (
- append_agents_to_model_group,
- append_agents_to_model_info,
-)
+from litellm.proxy._lazy_features import attach_lazy_features
from litellm.proxy.analytics_endpoints.analytics_endpoints import (
router as analytics_router,
)
-from litellm.proxy.anthropic_endpoints.claude_code_endpoints import (
- claude_code_marketplace_router,
-)
-from litellm.proxy.anthropic_endpoints.endpoints import router as anthropic_router
-from litellm.proxy.anthropic_endpoints.skills_endpoints import (
- router as anthropic_skills_router,
-)
from litellm.proxy.auth.auth_checks import (
ExperimentalUIJWTToken,
get_team_object,
@@ -328,7 +309,6 @@ from litellm.proxy.discovery_endpoints import ui_discovery_endpoints_router
from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router
from litellm.proxy.fine_tuning_endpoints.endpoints import set_fine_tuning_config
from litellm.proxy.google_endpoints.endpoints import router as google_router
-from litellm.proxy.guardrails.guardrail_endpoints import router as guardrails_router
from litellm.proxy.guardrails.init_guardrails import (
init_guardrails_v2,
initialize_guardrails,
@@ -344,9 +324,6 @@ from litellm.proxy.hooks.prompt_injection_detection import (
from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger
from litellm.proxy.image_endpoints.endpoints import router as image_router
from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
-from litellm.proxy.management_endpoints.access_group_endpoints import (
- router as access_group_router,
-)
from litellm.proxy.management_endpoints.budget_management_endpoints import (
router as budget_management_router,
)
@@ -358,14 +335,9 @@ from litellm.proxy.management_endpoints.callback_management_endpoints import (
)
from litellm.proxy.management_endpoints.common_utils import (
_user_has_admin_privileges,
+ _user_has_admin_view,
admin_can_invite_user,
)
-from litellm.proxy.management_endpoints.compliance_endpoints import (
- router as compliance_router,
-)
-from litellm.proxy.management_endpoints.config_override_endpoints import (
- router as config_override_router,
-)
from litellm.proxy.management_endpoints.cost_tracking_settings import (
router as cost_tracking_settings_router,
)
@@ -379,9 +351,6 @@ from litellm.proxy.management_endpoints.internal_user_endpoints import (
router as internal_user_router,
)
from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
-from litellm.proxy.management_endpoints.jwt_key_mapping_endpoints import (
- router as jwt_key_mapping_router,
-)
from litellm.proxy.management_endpoints.key_management_endpoints import (
delete_verification_tokens,
duration_in_seconds,
@@ -390,9 +359,6 @@ from litellm.proxy.management_endpoints.key_management_endpoints import (
from litellm.proxy.management_endpoints.key_management_endpoints import (
router as key_management_router,
)
-from litellm.proxy.management_endpoints.mcp_management_endpoints import (
- router as mcp_management_router,
-)
from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
router as model_access_group_management_router,
)
@@ -407,11 +373,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import (
from litellm.proxy.management_endpoints.organization_endpoints import (
router as organization_router,
)
-from litellm.proxy.management_endpoints.policy_endpoints import router as policy_router
from litellm.proxy.management_endpoints.router_settings_endpoints import (
router as router_settings_router,
)
-from litellm.proxy.management_endpoints.scim.scim_v2 import scim_router
from litellm.proxy.management_endpoints.tag_management_endpoints import (
router as tag_management_router,
)
@@ -423,9 +387,6 @@ from litellm.proxy.management_endpoints.team_endpoints import (
update_team,
validate_membership,
)
-from litellm.proxy.management_endpoints.tool_management_endpoints import (
- router as tool_management_router,
-)
from litellm.proxy.management_endpoints.workflow_management_endpoints import (
router as workflow_management_router,
)
@@ -434,7 +395,6 @@ from litellm.proxy.management_endpoints.ui_sso import (
get_disabled_non_admin_personal_key_creation,
)
from litellm.proxy.management_endpoints.ui_sso import router as ui_sso_router
-from litellm.proxy.management_endpoints.usage_endpoints import router as usage_ai_router
from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import (
router as user_agent_analytics_router,
)
@@ -444,7 +404,6 @@ from litellm.proxy.middleware.in_flight_requests_middleware import (
)
from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware
from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router
-from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_router
from litellm.proxy.openai_files_endpoints.files_endpoints import (
router as openai_files_router,
)
@@ -464,27 +423,16 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
router as pass_through_router,
)
-from litellm.proxy.policy_engine.policy_endpoints import router as policy_crud_router
-from litellm.proxy.policy_engine.policy_resolve_endpoints import (
- router as policy_resolve_router,
-)
-from litellm.proxy.prompts.prompt_endpoints import router as prompts_router
from litellm.proxy.public_endpoints import router as public_endpoints_router
from litellm.proxy.rag_endpoints.endpoints import router as rag_router
-from litellm.proxy.realtime_endpoints.endpoints import router as webrtc_router
from litellm.proxy.rerank_endpoints.endpoints import router as rerank_router
from litellm.proxy.response_api_endpoints.endpoints import router as response_router
from litellm.proxy.route_llm_request import route_request
from litellm.proxy.search_endpoints.endpoints import router as search_router
-from litellm.proxy.search_endpoints.search_tool_management import (
- router as search_tool_management_router,
-)
-from litellm.proxy.spend_tracking.cloudzero_endpoints import router as cloudzero_router
from litellm.proxy.spend_tracking.spend_management_endpoints import (
router as spend_management_router,
)
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
-from litellm.proxy.spend_tracking.vantage_endpoints import router as vantage_router
from litellm.proxy.types_utils.utils import get_instance_fn
from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import (
router as ui_crud_endpoints_router,
@@ -514,16 +462,6 @@ from litellm.proxy.utils import (
prefetch_config_params,
update_spend,
)
-from litellm.proxy.vector_store_endpoints.endpoints import router as vector_store_router
-from litellm.proxy.vector_store_endpoints.management_endpoints import (
- router as vector_store_management_router,
-)
-from litellm.proxy.vector_store_files_endpoints.endpoints import (
- router as vector_store_files_router,
-)
-from litellm.proxy.vertex_ai_endpoints.langfuse_endpoints import (
- router as langfuse_router,
-)
from litellm.proxy.video_endpoints.endpoints import router as video_router
from litellm.router import (
AssistantsTypedDict,
@@ -1019,6 +957,85 @@ async def proxy_startup_event(app: FastAPI): # noqa: PLR0915
await proxy_shutdown_event() # type: ignore[reportGeneralTypeIssues]
+def _generate_stable_operation_id(route: Any) -> str:
+ operation_id = re.sub(r"\W", "_", f"{route.name}{route.path_format}")
+ route_methods = sorted(route.methods or [])
+ if len(route_methods) == 1:
+ operation_id = f"{operation_id}_{route_methods[0].lower()}"
+ return operation_id
+
+
+_OPENAPI_HTTP_METHODS = {
+ "delete",
+ "get",
+ "head",
+ "options",
+ "patch",
+ "post",
+ "put",
+ "trace",
+}
+
+
+def _strip_operation_id_method_suffix(operation_id: str) -> str:
+ base, separator, suffix = operation_id.rpartition("_")
+ if separator and suffix in _OPENAPI_HTTP_METHODS:
+ return base
+ return operation_id
+
+
+def ensure_unique_openapi_operation_ids(
+ openapi_schema: Dict[str, Any],
+ reserved_operation_ids: Optional[Set[str]] = None,
+) -> Dict[str, Any]:
+ operation_entries = []
+ operation_id_counts: Dict[str, int] = {}
+ for path_item in openapi_schema.get("paths", {}).values():
+ if not isinstance(path_item, dict):
+ continue
+ for method, operation in path_item.items():
+ if method not in _OPENAPI_HTTP_METHODS or not isinstance(operation, dict):
+ continue
+ operation_id = operation.get("operationId")
+ if not isinstance(operation_id, str):
+ continue
+ operation_entries.append((method, operation, operation_id))
+ operation_id_counts[operation_id] = (
+ operation_id_counts.get(operation_id, 0) + 1
+ )
+
+ used_operation_ids = set(reserved_operation_ids or set())
+ seen_operation_ids: Set[str] = set()
+ for method, operation, operation_id in operation_entries:
+ should_rewrite = (
+ operation_id_counts[operation_id] > 1
+ or operation_id in used_operation_ids
+ or operation_id in seen_operation_ids
+ )
+ if not should_rewrite:
+ seen_operation_ids.add(operation_id)
+ used_operation_ids.add(operation_id)
+ continue
+
+ base_operation_id = _strip_operation_id_method_suffix(operation_id)
+ new_operation_id = f"{base_operation_id}_{method}"
+ suffix = 2
+ while (
+ new_operation_id in used_operation_ids
+ or new_operation_id in seen_operation_ids
+ ):
+ new_operation_id = f"{base_operation_id}_{method}_{suffix}"
+ suffix += 1
+ operation["operationId"] = new_operation_id
+ seen_operation_ids.add(new_operation_id)
+ used_operation_ids.add(new_operation_id)
+
+ if reserved_operation_ids is not None:
+ reserved_operation_ids.update(used_operation_ids)
+
+ return openapi_schema
+
+
app = FastAPI(
docs_url=_get_docs_url(),
redoc_url=_get_redoc_url(),
@@ -1028,6 +1045,7 @@ app = FastAPI(
version=version,
root_path=server_root_path,
lifespan=proxy_startup_event, # type: ignore[reportGeneralTypeIssues]
+ generate_unique_id_function=_generate_stable_operation_id,
)
vertex_live_passthrough_vertex_base = VertexBase()
@@ -1103,6 +1121,12 @@ def get_openapi_schema():
openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema)
+ # Stub unloaded lazy features so they appear as Swagger sections.
+ from litellm.proxy._lazy_features import inject_lazy_stubs
+
+ openapi_schema = inject_lazy_stubs(openapi_schema)
+ openapi_schema = ensure_unique_openapi_operation_ids(openapi_schema)
+
# Fix Swagger UI execute path error when server_root_path is set
if server_root_path:
openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}]
@@ -1129,6 +1153,12 @@ def custom_openapi():
openapi_schema = CustomOpenAPISpec.add_llm_api_request_schema_body(openapi_schema)
+ # Stub unloaded lazy features so they appear as Swagger sections.
+ from litellm.proxy._lazy_features import inject_lazy_stubs
+
+ openapi_schema = inject_lazy_stubs(openapi_schema)
+ openapi_schema = ensure_unique_openapi_operation_ids(openapi_schema)
+
# Fix Swagger UI execute path error when server_root_path is set
if server_root_path:
openapi_schema["servers"] = [{"url": "/" + server_root_path.strip("/")}]
@@ -1556,14 +1586,78 @@ def mount_swagger_ui():
app.mount("/swagger", StaticFiles(directory=swagger_directory), name="swagger")
+ # On dropdown expand: one-time fetch to the prefix (triggers lazy load),
+ # then spec re-download so real routes replace the stub. Raw JS (no
+ #