mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
Merge branch 'BerriAI:main' into dev
This commit is contained in:
NULL
GitHub
commit
c75e7230d2
3 changed files with 205 additions and 0 deletions
|
|
@ -1134,6 +1134,91 @@ When MCP tools are called, your custom hook will:
|
|||
2. Modify the response if needed
|
||||
3. Track costs in LiteLLM's logging system
|
||||
|
||||
## MCP Guardrails
|
||||
|
||||
LiteLLM supports applying guardrails to MCP tool calls to ensure security and compliance. You can configure guardrails to run before or during MCP calls to validate inputs and block or mask sensitive information.
|
||||
|
||||
### Supported MCP Guardrail Modes
|
||||
|
||||
MCP guardrails support the following modes:
|
||||
|
||||
- `pre_mcp_call`: Run **before** MCP call, on **input**. Use this mode when you want to apply validation/masking/blocking for MCP requests
|
||||
- `during_mcp_call`: Run **during** MCP call execution. Use this mode for real-time monitoring and intervention
|
||||
|
||||
### Configuration Examples
|
||||
|
||||
Configure guardrails to run before MCP tool calls to validate and sanitize inputs:
|
||||
|
||||
```yaml title="config.yaml" showLineNumbers
|
||||
guardrails:
|
||||
- guardrail_name: "mcp-input-validation"
|
||||
litellm_params:
|
||||
guardrail: presidio # or other supported guardrails
|
||||
mode: "pre_mcp_call" # or during_mcp_call
|
||||
pii_entities_config:
|
||||
CREDIT_CARD: "BLOCK" # Will block requests containing credit card numbers
|
||||
EMAIL_ADDRESS: "MASK" # Will mask email addresses
|
||||
PHONE_NUMBER: "MASK" # Will mask phone numbers
|
||||
default_on: true
|
||||
```
|
||||
|
||||
|
||||
### Usage Examples
|
||||
|
||||
#### Testing Pre-MCP Call Guardrails
|
||||
|
||||
Test your MCP guardrails with a request that includes sensitive information:
|
||||
|
||||
```bash title="Test MCP Guardrail" showLineNumbers
|
||||
curl http://localhost:4000/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "Authorization: Bearer sk-1234" \
|
||||
-d '{
|
||||
"model": "gpt-3.5-turbo",
|
||||
"messages": [
|
||||
{"role": "user", "content": "My credit card is 4111-1111-1111-1111 and my email is john@example.com"}
|
||||
],
|
||||
"guardrails": ["mcp-input-validation"]
|
||||
}'
|
||||
```
|
||||
|
||||
The request will be processed as follows:
|
||||
1. Credit card number will be blocked (request rejected)
|
||||
2. Email address will be masked (e.g., replaced with `<EMAIL_ADDRESS>`)
|
||||
|
||||
#### Using with MCP Tools
|
||||
|
||||
When using MCP tools, guardrails will be applied to the tool inputs:
|
||||
|
||||
```python title="Python Example with MCP Guardrails" showLineNumbers
|
||||
import openai
|
||||
|
||||
client = openai.OpenAI(
|
||||
api_key="your-api-key",
|
||||
base_url="http://localhost:4000"
|
||||
)
|
||||
|
||||
# This request will trigger MCP guardrails
|
||||
response = client.chat.completions.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[
|
||||
{"role": "user", "content": "Send an email to 555-123-4567 with my SSN 123-45-6789"}
|
||||
],
|
||||
tools=[{"type": "mcp", "server_label": "litellm", "server_url": "litellm_proxy"}],
|
||||
guardrails=["mcp-input-validation"]
|
||||
)
|
||||
```
|
||||
|
||||
### Supported Guardrail Providers
|
||||
|
||||
MCP guardrails work with all LiteLLM-supported guardrail providers:
|
||||
|
||||
- **Presidio**: PII detection and masking
|
||||
- **Bedrock**: AWS Bedrock guardrails
|
||||
- **Lakera**: Content moderation
|
||||
- **Aporia**: Custom guardrails
|
||||
- **Custom**: Your own guardrail implementations
|
||||
|
||||
## MCP Permission Management
|
||||
|
||||
LiteLLM supports managing permissions for MCP Servers by Keys, Teams, Organizations (entities) on LiteLLM. When a MCP client attempts to list tools, LiteLLM will only return the tools the entity has permissions to access.
|
||||
|
|
|
|||
|
|
@ -6074,6 +6074,36 @@
|
|||
"supports_tool_choice": true,
|
||||
"source": "https://inference-docs.cerebras.ai/support/pricing"
|
||||
},
|
||||
"cerebras/openai/gpt-oss-20b": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "cerebras",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://inference-docs.cerebras.ai/support/pricing"
|
||||
},
|
||||
"cerebras/openai/gpt-oss-120b": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"output_cost_per_token": 6.9e-07,
|
||||
"litellm_provider": "cerebras",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://www.cerebras.ai/blog/openai-gpt-oss-120b-runs-fastest-on-cerebras"
|
||||
},
|
||||
"friendliai/meta-llama-3.1-8b-instruct": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
|
|
@ -11658,6 +11688,36 @@
|
|||
"mode": "chat",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-oss-20b": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 8e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://openrouter.ai/openai/gpt-oss-20b"
|
||||
},
|
||||
"openrouter/openai/gpt-oss-120b": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 8e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://openrouter.ai/openai/gpt-oss-120b"
|
||||
},
|
||||
"openrouter/anthropic/claude-instant-v1": {
|
||||
"max_tokens": 100000,
|
||||
"max_output_tokens": 8191,
|
||||
|
|
|
|||
|
|
@ -6074,6 +6074,36 @@
|
|||
"supports_tool_choice": true,
|
||||
"source": "https://inference-docs.cerebras.ai/support/pricing"
|
||||
},
|
||||
"cerebras/openai/gpt-oss-20b": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 7e-08,
|
||||
"output_cost_per_token": 3e-07,
|
||||
"litellm_provider": "cerebras",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://inference-docs.cerebras.ai/support/pricing"
|
||||
},
|
||||
"cerebras/openai/gpt-oss-120b": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"output_cost_per_token": 6.9e-07,
|
||||
"litellm_provider": "cerebras",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://www.cerebras.ai/blog/openai-gpt-oss-120b-runs-fastest-on-cerebras"
|
||||
},
|
||||
"friendliai/meta-llama-3.1-8b-instruct": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
|
|
@ -11658,6 +11688,36 @@
|
|||
"mode": "chat",
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"openrouter/openai/gpt-oss-20b": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 8e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://openrouter.ai/openai/gpt-oss-20b"
|
||||
},
|
||||
"openrouter/openai/gpt-oss-120b": {
|
||||
"max_tokens": 32768,
|
||||
"max_input_tokens": 131072,
|
||||
"max_output_tokens": 32768,
|
||||
"input_cost_per_token": 1.8e-07,
|
||||
"output_cost_per_token": 8e-07,
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"source": "https://openrouter.ai/openai/gpt-oss-120b"
|
||||
},
|
||||
"openrouter/anthropic/claude-instant-v1": {
|
||||
"max_tokens": 100000,
|
||||
"max_output_tokens": 8191,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue