From beb426860dcb8bd41b2af692b1017a613afe01a9 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 17 Feb 2026 13:42:16 -0800 Subject: [PATCH] docs: document customer/end user object permission usage --- docs/my-website/docs/mcp.md | 62 ++++++ docs/my-website/docs/proxy/customers.md | 198 +++++++++++++++++- docs/my-website/docs/proxy/request_headers.md | 2 + litellm/proxy/litellm_pre_call_utils.py | 67 +++--- .../customer_endpoints.py | 51 ++++- 5 files changed, 345 insertions(+), 35 deletions(-) diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index 84d10c25931..8be507c6a6a 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -808,6 +808,68 @@ If your stdio MCP server needs per-request credentials, you can map HTTP headers In this example, when a client makes a request with the `X-GITHUB_PERSONAL_ACCESS_TOKEN` header, the proxy forwards that value into the stdio process as the `GITHUB_PERSONAL_ACCESS_TOKEN` environment variable. +## Control MCP Access for End Users + +Control which MCP servers end users of your AI application can access (e.g. users of an internal chat UI). Pass the customer ID in the `x-litellm-end-user` header to: +- Enforce object permissions (limit which MCP servers they can access) +- Apply customer-specific budgets +- Track spend per customer + +**FastMCP Client Example:** + +```python title="Track customer spend with x-litellm-end-user" showLineNumbers +from fastmcp import Client +import asyncio + +# MCP client configuration with customer tracking +config = { + "mcpServers": { + "github": { + "url": "http://localhost:4000/github_mcp/mcp", + "headers": { + "x-litellm-api-key": "Bearer sk-1234", + "x-litellm-end-user": "customer_123", # 👈 CUSTOMER ID + "Authorization": "Bearer gho_token" + } + } + } +} + +client = Client(config) + +async def main(): + async with client: + # All MCP calls will be tracked under customer_123 + tools = await client.list_tools() + result = await client.call_tool(tools[0].name, {}) + print(f"Tool result: {result}") + +asyncio.run(main()) +``` + +**Cursor IDE Example:** + +```json title="Cursor config with customer tracking" showLineNumbers +{ + "mcpServers": { + "GitHub": { + "url": "http://localhost:4000/github_mcp/mcp", + "headers": { + "x-litellm-api-key": "Bearer $LITELLM_API_KEY", + "x-litellm-end-user": "customer_123" + } + } + } +} +``` + +**What happens:** +- Customer-specific object permissions are enforced (only allowed MCP servers are accessible) +- Customer budgets are applied +- All tool calls are tracked under `customer_123` + +[Learn more about customer management →](./proxy/customers) + ## Using your MCP with client side credentials Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP. diff --git a/docs/my-website/docs/proxy/customers.md b/docs/my-website/docs/proxy/customers.md index 1101884c36b..8eb8aea6fb4 100644 --- a/docs/my-website/docs/proxy/customers.md +++ b/docs/my-website/docs/proxy/customers.md @@ -2,7 +2,7 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Customers / End-User Budgets +# Customers / End-Users Track spend, set budgets for your customers. @@ -10,9 +10,11 @@ Track spend, set budgets for your customers. ### 1. Make LLM API call w/ Customer ID -Make a /chat/completions call, pass 'user' - First call Works +You can pass the customer ID in two ways: -```bash showLineNumbers title="Make request with customer ID" +**Option 1: In the request body** (using the `user` field) + +```bash showLineNumbers title="Make request with customer ID in body" curl -X POST 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ --header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY @@ -28,6 +30,30 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ }' ``` +**Option 2: In the request headers** (using `x-litellm-end-user`) + +```bash showLineNumbers title="Make request with customer ID in header" +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'x-litellm-end-user: ishaan3' \ # 👈 CUSTOMER ID IN HEADER + --data ' { + "model": "azure-gpt-3.5", + "messages": [ + { + "role": "user", + "content": "what time is it" + } + ] + }' +``` + +**Use `x-litellm-end-user` to control permissions for end users of your AI application** (e.g. users of an internal chat UI): +- Apply customer-specific object permissions (limit which MCP servers they can access) +- Enforce customer budgets +- Works with all endpoints (chat/completions, embeddings, MCP, etc.) +- No need to modify request body + The customer_id will be upserted into the DB with the new spend. If the customer_id already exists, spend will be incremented. @@ -123,7 +149,171 @@ Expected Response -## Setting Customer Budgets +## Setting Customer Object Permissions + +Control which resources (MCP servers, vector stores, agents) a customer can access. + +### What are Object Permissions? + +Object permissions allow you to restrict customer access to specific: +- **MCP Servers**: Limit which MCP servers the customer can call +- **MCP Access Groups**: Assign customers to predefined groups of MCP servers +- **MCP Tool Permissions**: Granular control over which tools within an MCP server the customer can use +- **Vector Stores**: Control which vector stores the customer can query +- **Agents**: Restrict which agents the customer can interact with +- **Agent Access Groups**: Assign customers to predefined groups of agents + +### Creating a Customer with Object Permissions + +```bash showLineNumbers title="Create customer with object permissions" +curl -L -X POST 'http://localhost:4000/customer/new' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "user_id": "user_1", + "object_permission": { + "mcp_servers": ["server_1", "server_2"], + "mcp_access_groups": ["public_group"], + "mcp_tool_permissions": { + "server_1": ["tool_a", "tool_b"] + }, + "vector_stores": ["vector_store_1"], + "agents": ["agent_1"], + "agent_access_groups": ["basic_agents"] + } + }' +``` + +**Parameters:** +- `mcp_servers` (Optional[List[str]]): List of allowed MCP server IDs +- `mcp_access_groups` (Optional[List[str]]): List of MCP access group names +- `mcp_tool_permissions` (Optional[Dict[str, List[str]]]): Map of server ID to allowed tool names +- `vector_stores` (Optional[List[str]]): List of allowed vector store IDs +- `agents` (Optional[List[str]]): List of allowed agent IDs +- `agent_access_groups` (Optional[List[str]]): List of agent access group names + +**Note:** If `object_permission` is `null` or `{}`, the customer has no object-level restrictions. + +### Updating Customer Object Permissions + +You can update object permissions for existing customers: + +```bash showLineNumbers title="Update customer object permissions" +curl -L -X POST 'http://localhost:4000/customer/update' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "user_id": "user_1", + "object_permission": { + "mcp_servers": ["server_3"], + "vector_stores": ["vector_store_2", "vector_store_3"] + } + }' +``` + +### Viewing Customer Object Permissions + +When you query customer info, object permissions are included in the response: + +```bash showLineNumbers title="Get customer info with object permissions" +curl -X GET 'http://0.0.0.0:4000/customer/info?end_user_id=user_1' \ + -H 'Authorization: Bearer sk-1234' +``` + +**Response:** +```json showLineNumbers title="Response with object permissions" +{ + "user_id": "user_1", + "blocked": false, + "alias": "John Doe", + "spend": 0.0, + "object_permission": { + "object_permission_id": "perm_abc123", + "mcp_servers": ["server_1", "server_2"], + "mcp_access_groups": ["public_group"], + "mcp_tool_permissions": { + "server_1": ["tool_a", "tool_b"] + }, + "vector_stores": ["vector_store_1"], + "agents": ["agent_1"], + "agent_access_groups": ["basic_agents"] + }, + "litellm_budget_table": null +} +``` + +### Use Cases + +**1. Tiered Access Control** +Create different permission tiers for your customers: + +```bash showLineNumbers title="Free tier customer" +# Free tier - limited access +curl -L -X POST 'http://localhost:4000/customer/new' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "user_id": "free_user", + "budget_id": "free_tier", + "object_permission": { + "mcp_access_groups": ["public_group"], + "agent_access_groups": ["basic_agents"] + } + }' +``` + +```bash showLineNumbers title="Premium tier customer" +# Premium tier - full access +curl -L -X POST 'http://localhost:4000/customer/new' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "user_id": "premium_user", + "budget_id": "premium_tier", + "object_permission": { + "mcp_servers": ["server_1", "server_2", "server_3"], + "vector_stores": ["vector_store_1", "vector_store_2"], + "agents": ["agent_1", "agent_2", "agent_3"] + } + }' +``` + +**2. Department-Specific Access** +Restrict customers to resources relevant to their department: + +```bash showLineNumbers title="Sales team customer" +curl -L -X POST 'http://localhost:4000/customer/new' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "user_id": "sales_user", + "object_permission": { + "mcp_servers": ["crm_server", "email_server"], + "agents": ["sales_assistant"], + "vector_stores": ["sales_knowledge_base"] + } + }' +``` + +**3. Tool-Level Restrictions** +Grant access to specific tools within an MCP server: + +```bash showLineNumbers title="Limited tool access" +curl -L -X POST 'http://localhost:4000/customer/new' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "user_id": "restricted_user", + "object_permission": { + "mcp_servers": ["database_server"], + "mcp_tool_permissions": { + "database_server": ["read_only_query", "get_table_schema"] + } + } + }' +``` + +## Setting Customer Budgets Set customer budgets (e.g. monthly budgets, tpm/rpm limits) on LiteLLM Proxy diff --git a/docs/my-website/docs/proxy/request_headers.md b/docs/my-website/docs/proxy/request_headers.md index 090c201f884..b26d21823f2 100644 --- a/docs/my-website/docs/proxy/request_headers.md +++ b/docs/my-website/docs/proxy/request_headers.md @@ -20,6 +20,8 @@ By default, LiteLLM does not forward client headers to LLM provider APIs. Howeve `x-litellm-spend-logs-metadata`: Optional[str]: JSON string containing custom metadata to include in spend logs. Example: `{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}`. [Learn More](../proxy/enterprise#tracking-spend-with-custom-metadata) +`x-litellm-end-user`: Optional[str]: The customer/end-user ID to track spend and apply budgets/permissions. Alternative to passing `user` in the request body. Works with all endpoints including MCP. [Learn More](./customers) + ## Anthropic Headers `anthropic-version` Optional[str]: The version of the Anthropic API to use. diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 4d77af513a8..b1d4c8e507b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -10,15 +10,10 @@ import litellm from litellm._logging import verbose_logger, verbose_proxy_logger from litellm._service_logger import ServiceLogging from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from litellm.proxy._types import ( - AddTeamCallback, - CommonProxyErrors, - LitellmDataForBackendLLMCall, - LitellmUserRoles, - SpecialHeaders, - TeamCallbackMetadata, - UserAPIKeyAuth, -) +from litellm.proxy._types import (AddTeamCallback, CommonProxyErrors, + LitellmDataForBackendLLMCall, + LitellmUserRoles, SpecialHeaders, + TeamCallbackMetadata, UserAPIKeyAuth) # Cache special headers as a frozenset for O(1) lookup performance _SPECIAL_HEADERS_CACHE = frozenset( @@ -28,12 +23,9 @@ from litellm.proxy.auth.route_checks import RouteChecks from litellm.router import Router from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS from litellm.types.services import ServiceTypes -from litellm.types.utils import ( - LlmProviders, - ProviderSpecificHeader, - StandardLoggingUserAPIKeyMetadata, - SupportedCacheControls, -) +from litellm.types.utils import (LlmProviders, ProviderSpecificHeader, + StandardLoggingUserAPIKeyMetadata, + SupportedCacheControls) service_logger_obj = ServiceLogging() # used for tracking latency on OTEL @@ -395,6 +387,24 @@ class LiteLLMProxyRequestSetup: return user + @staticmethod + def get_end_user_from_headers(headers: dict) -> Optional[str]: + """ + Get the end user ID from the x-litellm-end-user header. + + This header allows you to track customer/end-user spend and apply customer-specific + budgets and permissions without modifying the request body. + + Returns: + Optional[str]: The end user ID if found in headers, None otherwise + """ + end_user = LiteLLMProxyRequestSetup._get_case_insensitive_header( + headers, "x-litellm-end-user" + ) + if end_user is not None: + verbose_logger.info(f'found end_user "{end_user}" in x-litellm-end-user header') + return end_user + @staticmethod def get_openai_org_id_from_headers( headers: dict, general_settings: Optional[Dict] = None @@ -640,8 +650,7 @@ class LiteLLMProxyRequestSetup: return data from litellm.proxy._types import ( LiteLLM_ManagementEndpoint_MetadataFields, - LiteLLM_ManagementEndpoint_MetadataFields_Premium, - ) + LiteLLM_ManagementEndpoint_MetadataFields_Premium) # ignore any special fields added_metadata = {} @@ -872,7 +881,15 @@ async def add_litellm_data_to_request( # noqa: PLR0915 general_settings, user_api_key_dict, _headers ) - # Parse user info from headers + # Parse end user ID from x-litellm-end-user header (takes precedence) + end_user_from_header = LiteLLMProxyRequestSetup.get_end_user_from_headers(_headers) + if end_user_from_header is not None: + if user_api_key_dict.end_user_id is None: + user_api_key_dict.end_user_id = end_user_from_header + if "user" not in data: + data["user"] = end_user_from_header + + # Parse user info from headers (fallback to general_settings.user_header_name) user = LiteLLMProxyRequestSetup.get_user_from_headers(_headers, general_settings) if user is not None: if user_api_key_dict.end_user_id is None: @@ -1530,12 +1547,9 @@ def _match_and_track_policies( """ from litellm._logging import verbose_proxy_logger from litellm.proxy.common_utils.callback_utils import ( - add_policy_sources_to_metadata, - add_policy_to_applied_policies_header, - ) - from litellm.proxy.policy_engine.attachment_registry import ( - get_attachment_registry, - ) + add_policy_sources_to_metadata, add_policy_to_applied_policies_header) + from litellm.proxy.policy_engine.attachment_registry import \ + get_attachment_registry from litellm.proxy.policy_engine.policy_matcher import PolicyMatcher # Get matching policies via attachments (with match reasons for attribution) @@ -1670,9 +1684,8 @@ def add_guardrails_from_policy_engine( user_api_key_dict: The user's API key authentication info """ from litellm._logging import verbose_proxy_logger - from litellm.proxy.common_utils.http_parsing_utils import ( - get_tags_from_request_body, - ) + from litellm.proxy.common_utils.http_parsing_utils import \ + get_tags_from_request_body from litellm.proxy.policy_engine.policy_registry import get_policy_registry from litellm.types.proxy.policy_engine import PolicyMatchContext diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index cd4aa9cbe91..bfafc943c45 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -233,7 +233,16 @@ async def new_end_user( - soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests. - spend: Optional[float] - Specify initial spend for a given customer. - budget_reset_at: Optional[str] - Specify the date and time when the budget should be reset. - - object_permission: Optional[LiteLLM_ObjectPermissionBase] - Customer-specific object permission. Example - {"mcp_servers": ["server_1", "server_2"], "vector_stores": ["vector_store_1"], "agents": ["agent_1"]}. IF null or {} then no object permission. + - object_permission: Optional[LiteLLM_ObjectPermissionBase] - Customer-specific object permissions to control access to resources. + Supported fields: + * mcp_servers: List[str] - List of allowed MCP server IDs + * mcp_access_groups: List[str] - List of MCP access group names + * mcp_tool_permissions: Dict[str, List[str]] - Map of server ID to allowed tool names (e.g., {"server_1": ["tool_a", "tool_b"]}) + * vector_stores: List[str] - List of allowed vector store IDs + * agents: List[str] - List of allowed agent IDs + * agent_access_groups: List[str] - List of agent access group names + Example: {"mcp_servers": ["server_1", "server_2"], "vector_stores": ["vector_store_1"], "agents": ["agent_1"]} + IF null or {} then no object-level restrictions apply. - Allow specifying allowed regions @@ -248,9 +257,22 @@ async def new_end_user( "user_id" : "ishaan-jaff-3", "allowed_region": "eu", "budget_id": "free_tier", - "default_model": "azure/gpt-3.5-turbo-eu" <- all calls from this user, use this model? + "default_model": "azure/gpt-3.5-turbo-eu" }' + # With object permissions + curl -L -X POST 'http://localhost:4000/customer/new' \ + -H 'Authorization: Bearer sk-1234' \ + -H 'Content-Type: application/json' \ + -d '{ + "user_id": "user_1", + "object_permission": { + "mcp_servers": ["server_1"], + "mcp_access_groups": ["public_group"], + "vector_stores": ["vector_store_1"] + } + }' + # return end-user object ``` @@ -461,7 +483,16 @@ async def update_end_user( - default_model: Optional[str] = ( None # if no equivalent model in allowed region - default all requests to this model ) - - object_permission: Optional[LiteLLM_ObjectPermissionBase] = Customer-specific object permission. Example - {"mcp_servers": ["server_1"], "vector_stores": ["vector_store_1"]}. IF null or {} then no object permission. + - object_permission: Optional[LiteLLM_ObjectPermissionBase] - Customer-specific object permissions to control access to resources. + Supported fields: + * mcp_servers: List[str] - List of allowed MCP server IDs + * mcp_access_groups: List[str] - List of MCP access group names + * mcp_tool_permissions: Dict[str, List[str]] - Map of server ID to allowed tool names + * vector_stores: List[str] - List of allowed vector store IDs + * agents: List[str] - List of allowed agent IDs + * agent_access_groups: List[str] - List of agent access group names + Example: {"mcp_servers": ["server_1"], "vector_stores": ["vector_store_1"]} + IF null or {} then no object-level restrictions apply. Example curl: ``` @@ -473,7 +504,19 @@ async def update_end_user( "budget_id": "paid_tier" }' - See below for all params + # Updating object permissions + curl -L -X POST 'http://localhost:4000/customer/update' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "user_id": "user_1", + "object_permission": { + "mcp_servers": ["server_3"], + "vector_stores": ["vector_store_2", "vector_store_3"] + } + }' + + See below for all params ``` """