From 035e5497e0e067b715df4a1b038d13d880676731 Mon Sep 17 00:00:00 2001 From: "Jugal D. Bhatt" <55304795+jugaldb@users.noreply.github.com> Date: Fri, 8 Aug 2025 22:47:31 -0700 Subject: [PATCH 1/2] added mcp guardrails doc in mcp.md (#13452) --- docs/my-website/docs/mcp.md | 85 +++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index 380a3b2be9c..684c2e6ca74 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -1134,6 +1134,91 @@ When MCP tools are called, your custom hook will: 2. Modify the response if needed 3. Track costs in LiteLLM's logging system +## MCP Guardrails + +LiteLLM supports applying guardrails to MCP tool calls to ensure security and compliance. You can configure guardrails to run before or during MCP calls to validate inputs and block or mask sensitive information. + +### Supported MCP Guardrail Modes + +MCP guardrails support the following modes: + +- `pre_mcp_call`: Run **before** MCP call, on **input**. Use this mode when you want to apply validation/masking/blocking for MCP requests +- `during_mcp_call`: Run **during** MCP call execution. Use this mode for real-time monitoring and intervention + +### Configuration Examples + +Configure guardrails to run before MCP tool calls to validate and sanitize inputs: + +```yaml title="config.yaml" showLineNumbers +guardrails: + - guardrail_name: "mcp-input-validation" + litellm_params: + guardrail: presidio # or other supported guardrails + mode: "pre_mcp_call" # or during_mcp_call + pii_entities_config: + CREDIT_CARD: "BLOCK" # Will block requests containing credit card numbers + EMAIL_ADDRESS: "MASK" # Will mask email addresses + PHONE_NUMBER: "MASK" # Will mask phone numbers + default_on: true +``` + + +### Usage Examples + +#### Testing Pre-MCP Call Guardrails + +Test your MCP guardrails with a request that includes sensitive information: + +```bash title="Test MCP Guardrail" showLineNumbers +curl http://localhost:4000/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": "My credit card is 4111-1111-1111-1111 and my email is john@example.com"} + ], + "guardrails": ["mcp-input-validation"] + }' +``` + +The request will be processed as follows: +1. Credit card number will be blocked (request rejected) +2. Email address will be masked (e.g., replaced with ``) + +#### Using with MCP Tools + +When using MCP tools, guardrails will be applied to the tool inputs: + +```python title="Python Example with MCP Guardrails" showLineNumbers +import openai + +client = openai.OpenAI( + api_key="your-api-key", + base_url="http://localhost:4000" +) + +# This request will trigger MCP guardrails +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": "Send an email to 555-123-4567 with my SSN 123-45-6789"} + ], + tools=[{"type": "mcp", "server_label": "litellm", "server_url": "litellm_proxy"}], + guardrails=["mcp-input-validation"] +) +``` + +### Supported Guardrail Providers + +MCP guardrails work with all LiteLLM-supported guardrail providers: + +- **Presidio**: PII detection and masking +- **Bedrock**: AWS Bedrock guardrails +- **Lakera**: Content moderation +- **Aporia**: Custom guardrails +- **Custom**: Your own guardrail implementations + ## MCP Permission Management LiteLLM supports managing permissions for MCP Servers by Keys, Teams, Organizations (entities) on LiteLLM. When a MCP client attempts to list tools, LiteLLM will only return the tools the entity has permissions to access. From d874bec480506afb99275044fe40e0c4cba37fce Mon Sep 17 00:00:00 2001 From: Cole McIntosh <82463175+colesmcintosh@users.noreply.github.com> Date: Fri, 8 Aug 2025 23:47:51 -0600 Subject: [PATCH 2/2] feat(models): add OpenRouter and Cerebras GPT-OSS models (20b, 120b) with pricing and context windows; update backup; refs #13428 (#13442) --- ...odel_prices_and_context_window_backup.json | 60 +++++++++++++++++++ model_prices_and_context_window.json | 60 +++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f9650127810..1bd7b460d6c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6074,6 +6074,36 @@ "supports_tool_choice": true, "source": "https://inference-docs.cerebras.ai/support/pricing" }, + "cerebras/openai/gpt-oss-20b": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 3e-07, + "litellm_provider": "cerebras", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://inference-docs.cerebras.ai/support/pricing" + }, + "cerebras/openai/gpt-oss-120b": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 6.9e-07, + "litellm_provider": "cerebras", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://www.cerebras.ai/blog/openai-gpt-oss-120b-runs-fastest-on-cerebras" + }, "friendliai/meta-llama-3.1-8b-instruct": { "max_tokens": 8192, "max_input_tokens": 8192, @@ -11658,6 +11688,36 @@ "mode": "chat", "supports_tool_choice": true }, + "openrouter/openai/gpt-oss-20b": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://openrouter.ai/openai/gpt-oss-20b" + }, + "openrouter/openai/gpt-oss-120b": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://openrouter.ai/openai/gpt-oss-120b" + }, "openrouter/anthropic/claude-instant-v1": { "max_tokens": 100000, "max_output_tokens": 8191, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f9650127810..1bd7b460d6c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6074,6 +6074,36 @@ "supports_tool_choice": true, "source": "https://inference-docs.cerebras.ai/support/pricing" }, + "cerebras/openai/gpt-oss-20b": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 7e-08, + "output_cost_per_token": 3e-07, + "litellm_provider": "cerebras", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://inference-docs.cerebras.ai/support/pricing" + }, + "cerebras/openai/gpt-oss-120b": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 6.9e-07, + "litellm_provider": "cerebras", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://www.cerebras.ai/blog/openai-gpt-oss-120b-runs-fastest-on-cerebras" + }, "friendliai/meta-llama-3.1-8b-instruct": { "max_tokens": 8192, "max_input_tokens": 8192, @@ -11658,6 +11688,36 @@ "mode": "chat", "supports_tool_choice": true }, + "openrouter/openai/gpt-oss-20b": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://openrouter.ai/openai/gpt-oss-20b" + }, + "openrouter/openai/gpt-oss-120b": { + "max_tokens": 32768, + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 8e-07, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://openrouter.ai/openai/gpt-oss-120b" + }, "openrouter/anthropic/claude-instant-v1": { "max_tokens": 100000, "max_output_tokens": 8191,