From 041ac054b6536810ae498fb1cc64e9c173fb25bb Mon Sep 17 00:00:00 2001 From: YutaSaito <36355491+uc4w6c@users.noreply.github.com> Date: Sat, 22 Nov 2025 01:52:01 +0900 Subject: [PATCH 01/29] feat: allow custom violation message for tool-permission guardrail (#16916) --- .../docs/proxy/guardrails/tool_permission.md | 43 ++++++++++++++-- litellm/integrations/custom_guardrail.py | 25 ++++++++- .../tool_permission_example.yaml | 3 +- .../guardrail_hooks/tool_permission.py | 22 ++++++-- .../guardrails/guardrail_initializers.py | 8 ++- litellm/types/guardrails.py | 36 ++++++++----- .../guardrail_hooks/test_tool_permission.py | 51 +++++++++++++++++++ 7 files changed, 164 insertions(+), 24 deletions(-) diff --git a/docs/my-website/docs/proxy/guardrails/tool_permission.md b/docs/my-website/docs/proxy/guardrails/tool_permission.md index 9ed05ed46a8..22ecdd2251e 100644 --- a/docs/my-website/docs/proxy/guardrails/tool_permission.md +++ b/docs/my-website/docs/proxy/guardrails/tool_permission.md @@ -46,6 +46,43 @@ guardrails: - `pre_call` Run **before** LLM call, on **input** - `post_call` Run **after** LLM call, on **input & output** +### `on_disallowed_action` behavior + +| Value | What happens | +| --- | --- | +| `block` | The request is immediately rejected. Pre-call checks raise a `400` HTTP error. Post-call checks raise `GuardrailRaisedException`, so the proxy responds with an error instead of the model output. Use when invoking the forbidden tool must halt the workflow. | +| `rewrite` | LiteLLM silently strips disallowed tools from the payload before it reaches the model (pre-call) or rewrites the model response/tool calls after the fact. The guardrail inserts error text into `message.content`/`tool_result` entries so the client learns the tool was blocked while the rest of the completion continues. Use when you want graceful degradation instead of hard failures. | + +### Custom denial message + +Set `violation_message_template` when you want the guardrail to return a branded error (e.g., “this violates our org policy…”). LiteLLM replaces placeholders from the denied tool: + +- `{tool_name}` – the tool/function name (e.g., `Read`) +- `{rule_id}` – the matching rule ID (or `None` when the default action kicks in) +- `{default_message}` – the original LiteLLM message if you need to append it + +Example: + +```yaml +guardrails: + - guardrail_name: "tool-permission-guardrail" + litellm_params: + guardrail: tool_permission + mode: "post_call" + violation_message_template: "this violates our org policy, we don't support executing {tool_name} commands" + rules: + - id: "allow_bash" + tool_name: "Bash" + decision: "allow" + - id: "deny_read" + tool_name: "Read" + decision: "deny" + default_action: "deny" + on_disallowed_action: "block" +``` + +If a request tries to invoke `Read`, the proxy now returns “this violates our org policy, we don't support executing Read commands” instead of the stock error text. Omit the field to keep the default messaging. + ### 2. Start the Proxy ```shell @@ -57,7 +94,7 @@ litellm --config config.yaml --port 4000 -**Block requset** +**Block request (`on_disallowed_action: block`)** ```bash # Test @@ -96,7 +133,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \ -**Rewrite requset** +**Rewrite request (`on_disallowed_action: rewrite`)** ```bash # Test @@ -118,7 +155,7 @@ curl -X POST "http://localhost:4000/v1/chat/completions" \ }' ``` -**Expected response:** +**Expected response (tool removed, completion continues):** ```json { diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index b50d05ed2ec..b52f1b3095e 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -36,6 +36,7 @@ class CustomGuardrail(CustomLogger): default_on: bool = False, mask_request_content: bool = False, mask_response_content: bool = False, + violation_message_template: Optional[str] = None, **kwargs, ): """ @@ -57,12 +58,34 @@ class CustomGuardrail(CustomLogger): self.default_on: bool = default_on self.mask_request_content: bool = mask_request_content self.mask_response_content: bool = mask_response_content + self.violation_message_template: Optional[str] = violation_message_template if supported_event_hooks: ## validate event_hook is in supported_event_hooks self._validate_event_hook(event_hook, supported_event_hooks) super().__init__(**kwargs) + def render_violation_message( + self, default: str, context: Optional[Dict[str, Any]] = None + ) -> str: + """Return a custom violation message if template is configured.""" + + if not self.violation_message_template: + return default + + format_context: Dict[str, Any] = {"default_message": default} + if context: + format_context.update(context) + try: + return self.violation_message_template.format(**format_context) + except Exception as e: + verbose_logger.warning( + "Failed to format violation message template for guardrail %s: %s", + self.guardrail_name, + e, + ) + return default + @staticmethod def get_config_model() -> Optional[Type["GuardrailConfigModel"]]: """ @@ -279,7 +302,7 @@ class CustomGuardrail(CustomLogger): data, self.event_hook ) if result is not None: - return result + return result return True def _event_hook_is_event_type(self, event_type: GuardrailEventHooks) -> bool: diff --git a/litellm/proxy/example_config_yaml/tool_permission_example.yaml b/litellm/proxy/example_config_yaml/tool_permission_example.yaml index e18425ba383..735b4bb7ed2 100644 --- a/litellm/proxy/example_config_yaml/tool_permission_example.yaml +++ b/litellm/proxy/example_config_yaml/tool_permission_example.yaml @@ -10,6 +10,7 @@ guardrails: guardrail: tool_permission mode: "post_call" default_on: true # Apply to all requests by default + violation_message_template: "this violates our org policy, we don't support executing {tool_name} commands" rules: - id: "allow_bash" tool_name: "Bash" @@ -33,4 +34,4 @@ general_settings: # Optional: Add logging configuration litellm_settings: success_callback: ["langfuse"] - failure_callback: ["langfuse"] \ No newline at end of file + failure_callback: ["langfuse"] diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py index 97f8dd76bd4..19060fa9d6d 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py +++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py @@ -120,13 +120,27 @@ class ToolPermissionGuardrail(CustomGuardrail): for rule in self.rules: if self._matches_pattern(tool_name, rule.tool_name): is_allowed = rule.decision == "allow" - message = f"Tool '{tool_name}' {'allowed' if is_allowed else 'denied'} by rule '{rule.id}'" + default_message = f"Tool '{tool_name}' {'allowed' if is_allowed else 'denied'} by rule '{rule.id}'" + message = self.render_violation_message( + default=default_message, + context={ + "tool_name": tool_name, + "rule_id": rule.id, + }, + ) verbose_proxy_logger.debug(message) return is_allowed, rule.id, message # No rule matched, use default action is_allowed = self.default_action == "allow" - message = f"Tool '{tool_name}' {'allowed' if is_allowed else 'denied'} by default action" + default_message = f"Tool '{tool_name}' {'allowed' if is_allowed else 'denied'} by default action" + message = self.render_violation_message( + default=default_message, + context={ + "tool_name": tool_name, + "rule_id": None, + }, + ) verbose_proxy_logger.debug(message) return is_allowed, None, message @@ -449,7 +463,9 @@ class ToolPermissionGuardrail(CustomGuardrail): verbose_proxy_logger.debug("Tool Permission Guardrail: Checking response") # Extract tool_calls from the response - tool_calls = self._extract_tool_calls_from_response(assembled_model_response) + tool_calls = self._extract_tool_calls_from_response( + assembled_model_response + ) if not tool_calls: verbose_proxy_logger.debug( diff --git a/litellm/proxy/guardrails/guardrail_initializers.py b/litellm/proxy/guardrails/guardrail_initializers.py index 6a5ba22419b..f2083e9c67e 100644 --- a/litellm/proxy/guardrails/guardrail_initializers.py +++ b/litellm/proxy/guardrails/guardrail_initializers.py @@ -135,6 +135,7 @@ def initialize_tool_permission(litellm_params: LitellmParams, guardrail: Guardra default_action=getattr(litellm_params, "default_action", "deny"), on_disallowed_action=getattr(litellm_params, "on_disallowed_action", "block"), default_on=litellm_params.default_on, + violation_message_template=litellm_params.violation_message_template, ) litellm.logging_callback_manager.add_litellm_callback(_tool_permission_callback) return _tool_permission_callback @@ -172,9 +173,12 @@ def initialize_panw_prisma_airs(litellm_params, guardrail): raise ValueError("PANW Prisma AIRS: profile_name is required") _panw_callback = PanwPrismaAirsHandler( - guardrail_name=guardrail.get("guardrail_name", "panw_prisma_airs"), # Use .get() with default + guardrail_name=guardrail.get( + "guardrail_name", "panw_prisma_airs" + ), # Use .get() with default api_key=litellm_params.api_key, - api_base=litellm_params.api_base or "https://service.api.aisecurity.paloaltonetworks.com/v1/scan/sync/request", + api_base=litellm_params.api_base + or "https://service.api.aisecurity.paloaltonetworks.com/v1/scan/sync/request", profile_name=litellm_params.profile_name, default_on=litellm_params.default_on, ) diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index cae9623b44b..f2b9d71cca6 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -16,7 +16,6 @@ from litellm.types.proxy.guardrails.guardrail_hooks.ibm import ( ) - """ Pydantic object defining how to set guardrails on litellm proxy @@ -51,7 +50,7 @@ class SupportedGuardrailIntegrations(Enum): OPENAI_MODERATION = "openai_moderation" NOMA = "noma" TOOL_PERMISSION = "tool_permission" - ZSCALER_AI_GUARD = "zscaler_ai_guard" + ZSCALER_AI_GUARD = "zscaler_ai_guard" JAVELIN = "javelin" ENKRYPTAI = "enkryptai" IBM_GUARDRAILS = "ibm_guardrails" @@ -432,7 +431,7 @@ class ZscalerAIGuardConfigModel(BaseModel): policy_id: Optional[int] = Field( default=None, - description="Policy ID for Zscaler AI Guard. Can also be set via ZSCALER_AI_GUARD_POLICY_ID environment variable" + description="Policy ID for Zscaler AI Guard. Can also be set via ZSCALER_AI_GUARD_POLICY_ID environment variable", ) send_user_api_key_alias: Optional[bool] = Field( default=False, description="Whether to send user_API_key_alias in headers" @@ -444,6 +443,7 @@ class ZscalerAIGuardConfigModel(BaseModel): default=False, description="Whether to send user_API_key_team_id in headers" ) + class JavelinGuardrailConfigModel(BaseModel): """Configuration parameters for the Javelin guardrail""" @@ -479,7 +479,8 @@ class BlockedWord(BaseModel): description="Action to take when keyword is detected (BLOCK or MASK)" ) description: Optional[str] = Field( - default=None, description="Optional description explaining why this keyword is sensitive" + default=None, + description="Optional description explaining why this keyword is sensitive", ) @@ -491,15 +492,15 @@ class ContentFilterPattern(BaseModel): ) pattern_name: Optional[str] = Field( default=None, - description="Name of prebuilt pattern (e.g., 'us_ssn', 'credit_card'). Required if pattern_type is 'prebuilt'" + description="Name of prebuilt pattern (e.g., 'us_ssn', 'credit_card'). Required if pattern_type is 'prebuilt'", ) pattern: Optional[str] = Field( default=None, - description="Custom regex pattern. Required if pattern_type is 'regex'" + description="Custom regex pattern. Required if pattern_type is 'regex'", ) name: Optional[str] = Field( default=None, - description="Name for this pattern (used in logging and error messages)" + description="Name for this pattern (used in logging and error messages)", ) action: ContentFilterAction = Field( description="Action to take when pattern matches (BLOCK or MASK)" @@ -511,15 +512,13 @@ class ContentFilterConfigModel(BaseModel): patterns: Optional[List[ContentFilterPattern]] = Field( default=None, - description="List of patterns (prebuilt or custom regex) to detect" + description="List of patterns (prebuilt or custom regex) to detect", ) blocked_words: Optional[List[BlockedWord]] = Field( - default=None, - description="List of blocked words with individual actions" + default=None, description="List of blocked words with individual actions" ) blocked_words_file: Optional[str] = Field( - default=None, - description="Path to YAML file containing blocked_words list" + default=None, description="Path to YAML file containing blocked_words list" ) @@ -575,6 +574,11 @@ class BaseLitellmParams(BaseModel): # works for new and patch update guardrails description="Optional field if guardrail requires a 'model' parameter", ) + violation_message_template: Optional[str] = Field( + default=None, + description="Custom message when a guardrail blocks an action. Supports placeholders like {tool_name}, {rule_id}, and {default_message}.", + ) + # Model Armor params template_id: Optional[str] = Field( default=None, description="The ID of your Model Armor template" @@ -613,7 +617,7 @@ class LitellmParams( GraySwanGuardrailConfigModel, NomaGuardrailConfigModel, ToolPermissionGuardrailConfigModel, - ZscalerAIGuardConfigModel, + ZscalerAIGuardConfigModel, JavelinGuardrailConfigModel, ContentFilterConfigModel, BaseLitellmParams, @@ -671,10 +675,12 @@ class GuardrailEventHooks(str, Enum): class DynamicGuardrailParams(TypedDict): extra_body: Dict[str, Any] + class GUARDRAIL_DEFINITION_LOCATION(str, Enum): DB = "db" CONFIG = "config" + class GuardrailInfoResponse(BaseModel): guardrail_id: Optional[str] = None guardrail_name: str @@ -682,7 +688,9 @@ class GuardrailInfoResponse(BaseModel): guardrail_info: Optional[Dict] = None created_at: Optional[datetime] = None updated_at: Optional[datetime] = None - guardrail_definition_location: GUARDRAIL_DEFINITION_LOCATION = GUARDRAIL_DEFINITION_LOCATION.CONFIG + guardrail_definition_location: GUARDRAIL_DEFINITION_LOCATION = ( + GUARDRAIL_DEFINITION_LOCATION.CONFIG + ) def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py index a9d87398217..8c88b22f60e 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py @@ -129,6 +129,24 @@ class TestToolPermissionGuardrail: assert rule_id is None assert "default" in (msg or "") + def test_check_tool_permission_custom_template(self): + guardrail = ToolPermissionGuardrail( + guardrail_name="custom-template", + rules=self.test_rules, + default_action="deny", + violation_message_template="custom {tool_name} {rule_id} :: {default_message}", + ) + + _, rule_id, message = guardrail._check_tool_permission("Read") + assert rule_id == "deny_read" + assert message.startswith("custom Read deny_read") + assert "Tool 'Read' denied" in message + + _, rule_id, message = guardrail._check_tool_permission("UnknownTool") + assert rule_id is None + assert message.startswith("custom UnknownTool None") + assert "Tool 'UnknownTool' denied by default action" in message + def test_extract_tool_calls_openai_format(self): tool_call = { "id": "call_123", @@ -224,6 +242,39 @@ class TestToolPermissionGuardrail: ) assert excinfo.value.status_code == 400 + @pytest.mark.asyncio + async def test_async_pre_call_hook_uses_custom_template(self): + guardrail = ToolPermissionGuardrail( + guardrail_name="custom-template", + rules=self.test_rules, + default_action="deny", + on_disallowed_action="block", + violation_message_template="blocked {tool_name} by policy", + ) + + data = { + "tools": [ + {"type": "function", "function": {"name": "Read"}}, + ] + } + user_api_key_dict = UserAPIKeyAuth() + cache = DualCache(default_in_memory_ttl=1) + + with patch.object(guardrail, "should_run_guardrail", return_value=True): + with pytest.raises(HTTPException) as excinfo: + await guardrail.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=cache, + data=data, + call_type="completion", + ) + + assert excinfo.value.status_code == 400 + assert ( + excinfo.value.detail.get("detection_message") + == "blocked Read by policy" + ) + @pytest.mark.asyncio async def test_async_pre_call_hook_rewrite_mode(self): guardrail = ToolPermissionGuardrail( From be9f2e026c24a108e740f8a19bd8cfc1ec535523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=84=E3=83=AB=E3=83=9E=E3=83=AB=E3=83=84=E3=83=A8?= =?UTF-8?q?=E3=82=B7?= <192761079+TsurumaruTsuyoshi@users.noreply.github.com> Date: Sat, 22 Nov 2025 00:52:25 +0800 Subject: [PATCH 02/29] feat(pricing): Add Gemini 3 Pro Preview for OpenRouter (#16905) --- model_prices_and_context_window.json | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b4b4f763860..273502ac22b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -19977,6 +19977,53 @@ "supports_tool_choice": true, "supports_vision": true }, + "openrouter/google/gemini-3-pro-preview": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "cache_creation_input_token_cost_above_200k_tokens": 2.5e-07, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_batches": 1e-06, + "litellm_provider": "openrouter", + "max_audio_length_hours": 8.4, + "max_audio_per_prompt": 1, + "max_images_per_prompt": 3000, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_pdf_size_mb": 30, + "max_tokens": 65535, + "max_video_length": 1, + "max_videos_per_prompt": 10, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_batches": 6e-06, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_video_input": true, + "supports_vision": true, + "supports_web_search": true + }, "openrouter/google/gemini-pro-1.5": { "input_cost_per_image": 0.00265, "input_cost_per_token": 2.5e-06, From b96179a07a31bb4d5e17e3e9ac3e89915723604e Mon Sep 17 00:00:00 2001 From: superpoussin22 Date: Fri, 21 Nov 2025 17:52:44 +0100 Subject: [PATCH 03/29] Upgrade pydantic to version 2.11.0 (#16909) Updated pydantic version to 2.11.0 for compatibility. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 45d7d529ea1..3133370a715 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,7 +55,7 @@ jinja2==3.1.6 # for prompt templates aiohttp==3.12.14 # for network calls aioboto3==13.4.0 # for async sagemaker calls tenacity==8.5.0 # for retrying requests, when litellm.num_retries set -pydantic==2.10.2 # proxy + openai req. +pydantic==2.11.0 # proxy + openai req. + mcp jsonschema==4.22.0 # validating json schema websockets==13.1.0 # for realtime API soundfile==0.12.1 # for audio file processing From 41566722af6a198ee2c2a11c99ac1acf7d258554 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 21 Nov 2025 08:53:18 -0800 Subject: [PATCH 04/29] [Feat] UI - Prompt Management - Allow testing prompts with Chat UI (#16898) * TestPromptRequest * add prompts/test endpoint for testing prompt * TestPromptTestEndpoint * feat: working v1 of this ui * workig prompt endpoints * add chat ui for prompts * add conversation panel * add init chat ui --- litellm/proxy/prompts/prompt_endpoints.py | 159 +++++++++++- litellm/types/proxy/prompt_endpoints.py | 10 + .../test_prompt_test_endpoint.py | 134 ++++++++++ .../prompt_editor_view/ConversationPanel.tsx | 21 -- .../conversation_panel/EmptyState.tsx | 22 ++ .../conversation_panel/MessageBubble.tsx | 115 ++++++++ .../conversation_panel/MessageInput.tsx | 71 +++++ .../conversation_panel/MessageList.tsx | 42 +++ .../conversation_panel/VariableInput.tsx | 44 ++++ .../conversation_panel/VariableWarning.tsx | 38 +++ .../conversation_panel/index.tsx | 78 ++++++ .../conversation_panel/types.ts | 16 ++ .../conversation_panel/useConversation.ts | 245 ++++++++++++++++++ .../prompts/prompt_editor_view/index.tsx | 8 +- 14 files changed, 978 insertions(+), 25 deletions(-) create mode 100644 litellm/types/proxy/prompt_endpoints.py create mode 100644 tests/proxy_unit_tests/test_prompt_test_endpoint.py delete mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/ConversationPanel.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/EmptyState.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageBubble.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageInput.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageList.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/VariableInput.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/VariableWarning.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/index.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/types.ts create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/useConversation.ts diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py index b4677ed374b..373f993d3e3 100644 --- a/litellm/proxy/prompts/prompt_endpoints.py +++ b/litellm/proxy/prompts/prompt_endpoints.py @@ -6,7 +6,15 @@ import tempfile from pathlib import Path from typing import Any, Dict, List, Optional, cast -from fastapi import APIRouter, Depends, File, HTTPException, UploadFile +from fastapi import ( + APIRouter, + Depends, + File, + HTTPException, + Request, + Response, + UploadFile, +) from pydantic import BaseModel from litellm._logging import verbose_proxy_logger @@ -20,6 +28,7 @@ from litellm.types.prompts.init_prompts import ( PromptSpec, PromptTemplateBase, ) +from litellm.types.proxy.prompt_endpoints import TestPromptRequest router = APIRouter() @@ -667,6 +676,154 @@ async def patch_prompt( raise HTTPException(status_code=500, detail=str(e)) +@router.post( + "/prompts/test", + tags=["Prompt Management"], + dependencies=[Depends(user_api_key_auth)], +) +async def test_prompt( + request: TestPromptRequest, + fastapi_request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Test a prompt by rendering it with variables and executing an LLM call. + + This endpoint allows testing prompts before saving them to the database. + The response is always streamed. + + 👉 [Prompt docs](https://docs.litellm.ai/docs/proxy/prompt_management) + + Example Request: + ```bash + curl -X POST "http://localhost:4000/prompts/test" \\ + -H "Authorization: Bearer " \\ + -H "Content-Type: application/json" \\ + -d '{ + "dotprompt_content": "---\\nmodel: gpt-4o\\ntemperature: 0.7\\n---\\n\\nUser: Hello {{name}}", + "prompt_variables": { + "name": "World" + } + }' + ``` + """ + from pydantic import BaseModel + + from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager + from litellm.integrations.dotprompt.prompt_manager import ( + PromptManager, + PromptTemplate, + ) + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + try: + # Parse the dotprompt content and create PromptTemplate + prompt_manager = PromptManager() + frontmatter, template_content = prompt_manager._parse_frontmatter( + content=request.dotprompt_content + ) + + # Create PromptTemplate to leverage existing parameter extraction logic + template = PromptTemplate( + content=template_content, + metadata=frontmatter, + template_id="test_prompt" + ) + + # Extract model from template + if not template.model: + raise HTTPException( + status_code=400, + detail="Model is required in dotprompt metadata" + ) + + # Always render the template to extract system messages and other metadata + variables = request.prompt_variables or {} + rendered_content = prompt_manager.jinja_env.from_string( + template_content + ).render(**variables) + + # Convert rendered content to messages using DotpromptManager's method + dotprompt_manager = DotpromptManager() + rendered_messages = dotprompt_manager._convert_to_messages( + rendered_content=rendered_content + ) + + if not rendered_messages: + raise HTTPException( + status_code=400, + detail="No messages found in rendered prompt" + ) + + # If conversation history is provided, use it but preserve system messages + if request.conversation_history: + # Extract system messages from rendered prompt + system_messages = [msg for msg in rendered_messages if msg.get("role") == "system"] + # Use conversation history for user/assistant messages + messages = system_messages + request.conversation_history + else: + messages = rendered_messages + + # Use PromptTemplate's optional_params which already extracts all parameters + optional_params = template.optional_params.copy() + + # Always stream the response + optional_params["stream"] = True + + # Build request data for chat completion + data = { + "model": template.model, + "messages": messages, + } + data.update(optional_params) + + # Use ProxyBaseLLMRequestProcessing to go through all proxy logic + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + result = await base_llm_response_processor.base_process_llm_request( + request=fastapi_request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="acompletion", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + + if isinstance(result, BaseModel): + return result.model_dump(exclude_none=True, exclude_unset=True) + else: + return result + + except HTTPException as e: + raise e + except Exception as e: + verbose_proxy_logger.exception(f"Error testing prompt: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + @router.post( "/utils/dotprompt_json_converter", tags=["prompts", "utils"], diff --git a/litellm/types/proxy/prompt_endpoints.py b/litellm/types/proxy/prompt_endpoints.py new file mode 100644 index 00000000000..620a565b0a2 --- /dev/null +++ b/litellm/types/proxy/prompt_endpoints.py @@ -0,0 +1,10 @@ +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel + + +class TestPromptRequest(BaseModel): + dotprompt_content: str + prompt_variables: Optional[Dict[str, Any]] = None + conversation_history: Optional[List[Dict[str, str]]] = None + diff --git a/tests/proxy_unit_tests/test_prompt_test_endpoint.py b/tests/proxy_unit_tests/test_prompt_test_endpoint.py new file mode 100644 index 00000000000..327f60e3d7b --- /dev/null +++ b/tests/proxy_unit_tests/test_prompt_test_endpoint.py @@ -0,0 +1,134 @@ +""" +Test /prompts/test endpoint for testing prompts before saving +""" + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch +from fastapi import HTTPException + + +class TestPromptTestEndpoint: + """ + Tests the /prompts/test endpoint that allows testing prompts with variables + """ + + @pytest.mark.asyncio + async def test_parse_dotprompt_with_variables(self): + """ + Test that dotprompt content is parsed and variables are rendered correctly + """ + from litellm.integrations.dotprompt.prompt_manager import PromptManager + + dotprompt_content = """--- +model: gpt-4o +temperature: 0.7 +max_tokens: 100 +--- + +User: Hello {{name}}, how are you?""" + + # Parse the dotprompt + prompt_manager = PromptManager() + frontmatter, template_content = prompt_manager._parse_frontmatter( + content=dotprompt_content + ) + + assert frontmatter["model"] == "gpt-4o" + assert frontmatter["temperature"] == 0.7 + assert frontmatter["max_tokens"] == 100 + assert "{{name}}" in template_content + + # Render with variables + from jinja2 import Environment + + jinja_env = Environment( + variable_start_string="{{", + variable_end_string="}}", + ) + jinja_template = jinja_env.from_string(template_content) + rendered = jinja_template.render(name="World") + + assert "Hello World" in rendered + assert "{{name}}" not in rendered + + @pytest.mark.asyncio + async def test_convert_to_messages_format(self): + """ + Test that rendered prompt is converted to OpenAI messages format + """ + import re + + rendered_content = """System: You are a helpful assistant. + +User: Hello World, how are you?""" + + messages = [] + role_pattern = r"^(System|User|Assistant|Developer):\s*(.*?)(?=\n(?:System|User|Assistant|Developer):|$)" + matches = list( + re.finditer( + pattern=role_pattern, + string=rendered_content.strip(), + flags=re.MULTILINE | re.DOTALL, + ) + ) + + for match in matches: + role = match.group(1).lower() + content = match.group(2).strip() + + if role == "developer": + role = "system" + + if content: + messages.append({"role": role, "content": content}) + + assert len(messages) == 2 + assert messages[0]["role"] == "system" + assert "helpful assistant" in messages[0]["content"] + assert messages[1]["role"] == "user" + assert "Hello World" in messages[1]["content"] + + @pytest.mark.asyncio + async def test_single_message_without_role(self): + """ + Test that content without role markers is treated as a user message + """ + import re + + rendered_content = "Just a plain message without any role markers" + + messages = [] + role_pattern = r"^(System|User|Assistant|Developer):\s*(.*?)(?=\n(?:System|User|Assistant|Developer):|$)" + matches = list( + re.finditer( + pattern=role_pattern, + string=rendered_content.strip(), + flags=re.MULTILINE | re.DOTALL, + ) + ) + + if not matches: + messages.append({"role": "user", "content": rendered_content.strip()}) + + assert len(messages) == 1 + assert messages[0]["role"] == "user" + assert messages[0]["content"] == rendered_content + + @pytest.mark.asyncio + async def test_missing_model_raises_error(self): + """ + Test that missing model in frontmatter raises an error + """ + from litellm.integrations.dotprompt.prompt_manager import PromptManager + + dotprompt_content = """--- +temperature: 0.7 +--- + +User: Hello""" + + prompt_manager = PromptManager() + frontmatter, _ = prompt_manager._parse_frontmatter(content=dotprompt_content) + + model = frontmatter.get("model") + assert model is None diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/ConversationPanel.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/ConversationPanel.tsx deleted file mode 100644 index a827349d6e1..00000000000 --- a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/ConversationPanel.tsx +++ /dev/null @@ -1,21 +0,0 @@ -import React from "react"; -import { MessageSquareIcon } from "lucide-react"; - -const ConversationPanel: React.FC = () => { - return ( -
-
-
-
- -
-

Your conversation will appear here

-

Save the prompt to test it

-
-
-
- ); -}; - -export default ConversationPanel; - diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/EmptyState.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/EmptyState.tsx new file mode 100644 index 00000000000..57a6a54760d --- /dev/null +++ b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/EmptyState.tsx @@ -0,0 +1,22 @@ +import React from "react"; +import { RobotOutlined } from "@ant-design/icons"; + +interface EmptyStateProps { + hasVariables: boolean; +} + +const EmptyState: React.FC = ({ hasVariables }) => { + return ( +
+ + + {hasVariables + ? "Fill in the variables above, then type a message to start testing" + : "Type a message below to start testing your prompt"} + +
+ ); +}; + +export default EmptyState; + diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageBubble.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageBubble.tsx new file mode 100644 index 00000000000..e5e4f46b409 --- /dev/null +++ b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageBubble.tsx @@ -0,0 +1,115 @@ +import React from "react"; +import { RobotOutlined, UserOutlined } from "@ant-design/icons"; +import ReactMarkdown from "react-markdown"; +import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; +import { coy } from "react-syntax-highlighter/dist/esm/styles/prism"; +import ResponseMetrics from "../../../playground/chat_ui/ResponseMetrics"; +import { Message } from "./types"; + +interface MessageBubbleProps { + message: Message; +} + +const MessageBubble: React.FC = ({ message }) => { + return ( +
+
+
+
+ {message.role === "user" ? ( + + ) : ( + + )} +
+ {message.role} + {message.role === "assistant" && message.model && ( + + {message.model} + + )} +
+ +
+ {message.role === "assistant" ? ( + & { + inline?: boolean; + node?: any; + }) { + const match = /language-(\w+)/.exec(className || ""); + return !inline && match ? ( + + {String(children).replace(/\n$/, "")} + + ) : ( + + {children} + + ); + }, + pre: ({ node, ...props }) => ( + 
+                ),
+              }}
+            >
+              {message.content}
+            
+          ) : (
+            
{message.content}

+          )}
+
+          {message.role === "assistant" &&
+            (message.timeToFirstToken || message.totalLatency || message.usage) && (
+              
+            )}
+
+
+
+ ); +}; + +export default MessageBubble; + diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageInput.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageInput.tsx new file mode 100644 index 00000000000..9ec232dee40 --- /dev/null +++ b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/conversation_panel/MessageInput.tsx @@ -0,0 +1,71 @@ +import React from "react"; +import { ArrowUpOutlined } from "@ant-design/icons"; +import { Button as TremorButton } from "@tremor/react"; +import { Input } from "antd"; + +const { TextArea } = Input; + +interface MessageInputProps { + inputMessage: string; + isLoading: boolean; + isDisabled: boolean; + onInputChange: (value: string) => void; + onSend: () => void; + onKeyDown: (event: React.KeyboardEvent) => void; + onCancel: () => void; +} + +const MessageInput: React.FC = ({ + inputMessage, + isLoading, + isDisabled, + onInputChange, + onSend, + onKeyDown, + onCancel, +}) => { + return ( +
+
+