From e1f54ef02ca2c6972c3b94aaf52149cb55128188 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 27 Oct 2025 10:26:00 -0700 Subject: [PATCH 01/91] docs: refactor placement of adding guardrails to endpoints doc --- docs/my-website/sidebars.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 4e217b939de..82ef79c7870 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -32,6 +32,7 @@ const sidebars = { items: [ "proxy/guardrails/quick_start", ...[ + "adding_provider/adding_guardrail_support", "proxy/guardrails/aim_security", "proxy/guardrails/aporia_api", "proxy/guardrails/azure_content_guardrail", @@ -713,7 +714,6 @@ const sidebars = { items: [ "adding_provider/directory_structure", "adding_provider/new_rerank_provider", - "adding_provider/adding_guardrail_support"], }, "extras/contributing", "contributing", From 49b9bd3cada5bf60e3a3a5141804de0fbd25d31e Mon Sep 17 00:00:00 2001 From: Rob Geada Date: Mon, 27 Oct 2025 20:00:32 +0000 Subject: [PATCH 02/91] Update IBM Guardrails implementation to correctly registrer SSL Verify argument (#15975) Signed-off-by: Rob Geada --- .../guardrails/guardrail_hooks/ibm_guardrails/ibm_detector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/ibm_guardrails/ibm_detector.py b/litellm/proxy/guardrails/guardrail_hooks/ibm_guardrails/ibm_detector.py index 8a6b6d73978..e7e5beb0dba 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/ibm_guardrails/ibm_detector.py +++ b/litellm/proxy/guardrails/guardrail_hooks/ibm_guardrails/ibm_detector.py @@ -46,7 +46,8 @@ class IBMGuardrailDetector(CustomGuardrail): **kwargs, ): self.async_handler = get_async_httpx_client( - llm_provider=httpxSpecialProvider.GuardrailCallback + llm_provider=httpxSpecialProvider.GuardrailCallback, + params={"ssl_verify": verify_ssl} ) # Set API configuration From 4758e2998b2c6eda2cd0299c9bb3541b903144ce Mon Sep 17 00:00:00 2001 From: pinkgu <39329656+bjornjee@users.noreply.github.com> Date: Tue, 28 Oct 2025 04:34:59 +0800 Subject: [PATCH 03/91] feat: support during_call for model armor guardrails (#15970) Signed-off-by: bjornjee --- .../model_armor/model_armor.py | 123 ++++++- .../guardrail_hooks/test_model_armor.py | 327 +++++++++++++++++- 2 files changed, 444 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 82245f2ce0a..47c6cb7e01a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -226,7 +226,9 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): } } - def _should_block_content(self, armor_response: dict, allow_sanitization: bool = False) -> bool: + def _should_block_content( + self, armor_response: dict, allow_sanitization: bool = False + ) -> bool: """Check if Model Armor response indicates content should be blocked, including both inspectResult and deidentifyResult.""" sanitization_result = armor_response.get("sanitizationResult", {}) filter_results = sanitization_result.get("filterResults", {}) @@ -413,11 +415,15 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): # fail_on_error=False) we still want the correct status reflected. metadata["_model_armor_status"] = ( "blocked" - if self._should_block_content(armor_response, allow_sanitization=self.mask_request_content) + if self._should_block_content( + armor_response, allow_sanitization=self.mask_request_content + ) else "success" ) # Check if content should be blocked - if self._should_block_content(armor_response, allow_sanitization=self.mask_request_content): + if self._should_block_content( + armor_response, allow_sanitization=self.mask_request_content + ): raise HTTPException( status_code=400, detail={ @@ -456,6 +462,109 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): return data + @log_guardrail_information + async def async_moderation_hook( + self, + data: dict, + user_api_key_dict: UserAPIKeyAuth, + call_type: Literal[ + "completion", + "embeddings", + "image_generation", + "moderation", + "audio_transcription", + "responses", + "mcp_call", + "anthropic_messages", + ], + ) -> Union[Exception, str, dict, None]: + """During-call hook to sanitize user prompts in parallel with LLM call.""" + verbose_proxy_logger.debug("Inside Model Armor Moderation Hook") + + from litellm.proxy.common_utils.callback_utils import ( + add_guardrail_to_applied_guardrails_header, + ) + + event_type = GuardrailEventHooks.during_call + if self.should_run_guardrail(data=data, event_type=event_type) is not True: + return data + + messages = data.get("messages") + if not messages: + verbose_proxy_logger.warning( + "Model Armor: not running guardrail. No messages in data" + ) + return data + + # Extract content from messages + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + get_last_user_message, + ) + + content = get_last_user_message(messages) + if not content: + return data + + # Make Model Armor request + try: + armor_response = await self.make_model_armor_request( + content=content, + source="user_prompt", + request_data=data, + ) + + # Store the armor response for logging + if isinstance(data, dict): + metadata = data.setdefault("metadata", {}) + metadata["_model_armor_response"] = armor_response + metadata["_model_armor_status"] = ( + "blocked" + if self._should_block_content( + armor_response, allow_sanitization=self.mask_request_content + ) + else "success" + ) + + # Check if content should be blocked + if self._should_block_content( + armor_response, allow_sanitization=self.mask_request_content + ): + raise HTTPException( + status_code=400, + detail={ + "error": "Content blocked by Model Armor", + "model_armor_response": armor_response, + }, + ) + + # If mask_request_content is enabled, update messages with sanitized content + if self.mask_request_content: + sanitized_content = self._get_sanitized_content(armor_response) + if sanitized_content and sanitized_content != content: + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + set_last_user_message, + ) + + data["messages"] = set_last_user_message( + messages, sanitized_content + ) + + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.error( + "Model Armor moderation error: %s", str(e), exc_info=True + ) + if self.optional_params.get("fail_on_error", True): + raise + + # Add guardrail to headers + add_guardrail_to_applied_guardrails_header( + request_data=data, guardrail_name=self.guardrail_name + ) + + return data + @log_guardrail_information async def async_post_call_success_hook( self, @@ -498,12 +607,16 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): metadata["_model_armor_response"] = armor_response metadata["_model_armor_status"] = ( "blocked" - if self._should_block_content(armor_response, allow_sanitization=self.mask_response_content) + if self._should_block_content( + armor_response, allow_sanitization=self.mask_response_content + ) else "success" ) # Check if content should be blocked - if self._should_block_content(armor_response, allow_sanitization=self.mask_response_content): + if self._should_block_content( + armor_response, allow_sanitization=self.mask_response_content + ): raise HTTPException( status_code=400, detail={ diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py index b596d427c5d..ae0f8ec67ba 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_model_armor.py @@ -1166,4 +1166,329 @@ async def test_model_armor_with_default_credentials(): # Verify the project_id was used correctly in the API call guardrail.async_handler.post.assert_called_once() call_args = guardrail.async_handler.post.call_args - assert "cloud-test-project" in call_args[1]["url"] \ No newline at end of file + assert "cloud-test-project" in call_args[1]["url"] + + +# ===== ASYNC MODERATION HOOK TESTS ===== + +@pytest.mark.asyncio +async def test_async_moderation_hook_success_no_blocking(): + """Test async_moderation_hook with successful response (no blocking)""" + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + ) + + # Mock successful (no match found) response + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.json = AsyncMock(return_value={ + "sanitizationResult": { + "filterMatchState": "NO_MATCH_FOUND", + "filterResults": { + "rai": { + "raiFilterResult": { + "matchState": "NO_MATCH_FOUND" + } + } + } + } + }) + + # Mock the access token method and async handler + guardrail._ensure_access_token_async = AsyncMock(return_value=("test-token", "test-project")) + guardrail.async_handler = AsyncMock() + guardrail.async_handler.post = AsyncMock(return_value=mock_response) + + request_data = { + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "Hello, how are you?"} + ], + "metadata": {"guardrails": ["model-armor-test"]} + } + + result = await guardrail.async_moderation_hook( + data=request_data, + user_api_key_dict=mock_user_api_key_dict, + call_type="completion" + ) + + # Should return the original data unchanged + assert result == request_data + # Should have metadata added + assert "_model_armor_response" in request_data["metadata"] + assert request_data["metadata"]["_model_armor_status"] == "success" + + +@pytest.mark.asyncio +async def test_async_moderation_hook_content_blocked(): + """Test async_moderation_hook when content should be blocked""" + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + ) + + # Mock response that indicates content should be blocked + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.json = AsyncMock(return_value={ + "sanitizationResult": { + "filterMatchState": "MATCH_FOUND", + "filterResults": { + "rai": { + "raiFilterResult": { + "matchState": "MATCH_FOUND" + } + } + } + } + }) + + # Mock the access token method and async handler + guardrail._ensure_access_token_async = AsyncMock(return_value=("test-token", "test-project")) + guardrail.async_handler = AsyncMock() + guardrail.async_handler.post = AsyncMock(return_value=mock_response) + + request_data = { + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "Some harmful content"} + ], + "metadata": {"guardrails": ["model-armor-test"]} + } + + # Should raise HTTPException for blocked content + with pytest.raises(HTTPException) as exc_info: + await guardrail.async_moderation_hook( + data=request_data, + user_api_key_dict=mock_user_api_key_dict, + call_type="completion" + ) + + assert exc_info.value.status_code == 400 + assert "Content blocked by Model Armor" in str(exc_info.value.detail) + # Should have metadata added even when blocked + assert "_model_armor_response" in request_data["metadata"] + assert request_data["metadata"]["_model_armor_status"] == "blocked" + + +@pytest.mark.asyncio +async def test_async_moderation_hook_with_sanitization(): + """Test async_moderation_hook with content sanitization enabled""" + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + mask_request_content=True, # Enable sanitization + ) + + # Mock response with sanitized content + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.json = AsyncMock(return_value={ + "sanitizationResult": { + "filterMatchState": "MATCH_FOUND", + "filterResults": { + "sdp": { + "sdpFilterResult": { + "deidentifyResult": { + "matchState": "MATCH_FOUND", + "data": { + "text": "Hello, my phone number is [REDACTED]" + } + } + } + } + } + } + }) + + # Mock the access token method and async handler + guardrail._ensure_access_token_async = AsyncMock(return_value=("test-token", "test-project")) + guardrail.async_handler = AsyncMock() + guardrail.async_handler.post = AsyncMock(return_value=mock_response) + + original_content = "Hello, my phone number is 555-123-4567" + request_data = { + "model": "gpt-4", + "messages": [ + {"role": "user", "content": original_content} + ], + "metadata": {"guardrails": ["model-armor-test"]} + } + + result = await guardrail.async_moderation_hook( + data=request_data, + user_api_key_dict=mock_user_api_key_dict, + call_type="completion" + ) + + # Should return data with sanitized content + assert result == request_data + # Content should be sanitized + from litellm.litellm_core_utils.prompt_templates.common_utils import get_last_user_message + sanitized_content = get_last_user_message(request_data["messages"]) + assert sanitized_content == "Hello, my phone number is [REDACTED]" + assert sanitized_content != original_content + # Should have metadata added + assert "_model_armor_response" in request_data["metadata"] + assert request_data["metadata"]["_model_armor_status"] == "success" + + +@pytest.mark.asyncio +async def test_async_moderation_hook_no_user_messages(): + """Test async_moderation_hook when there are no user messages to check""" + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + ) + + request_data = { + "model": "gpt-4", + "messages": [ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "assistant", "content": "How can I help you?"} + ], + "metadata": {"guardrails": ["model-armor-test"]} + } + + result = await guardrail.async_moderation_hook( + data=request_data, + user_api_key_dict=mock_user_api_key_dict, + call_type="completion" + ) + + # Should return the original data unchanged since no user messages to check + assert result == request_data + + +@pytest.mark.asyncio +async def test_async_moderation_hook_should_not_run(): + """Test async_moderation_hook when guardrail should not run due to missing guardrail name""" + try: + import google.auth + except ImportError: + pytest.skip("google.auth not installed") + return + + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="different-guardrail-name", # Different name than what's in metadata + ) + + # Request data with a different guardrail name + request_data = { + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "Hello, how are you?"} + ], + "metadata": {"guardrails": ["some-other-guardrail"]} # Different guardrail name + } + + result = await guardrail.async_moderation_hook( + data=request_data, + user_api_key_dict=mock_user_api_key_dict, + call_type="completion" + ) + + # Should return the original data unchanged since guardrail name doesn't match + assert result == request_data + + +@pytest.mark.asyncio +async def test_async_moderation_hook_api_error_fail_on_error_true(): + """Test async_moderation_hook when API call fails and fail_on_error is True""" + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + optional_params={"fail_on_error": True} + ) + + # Mock the access token method + guardrail._ensure_access_token_async = AsyncMock(return_value=("test-token", "test-project")) + + # Mock the async handler to raise an exception + guardrail.async_handler = AsyncMock() + guardrail.async_handler.post = AsyncMock(side_effect=Exception("API Error")) + + request_data = { + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "Hello, how are you?"} + ], + "metadata": {"guardrails": ["model-armor-test"]} + } + + # Should raise the exception since fail_on_error is True + with pytest.raises(Exception) as exc_info: + await guardrail.async_moderation_hook( + data=request_data, + user_api_key_dict=mock_user_api_key_dict, + call_type="completion" + ) + + assert "API Error" in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_async_moderation_hook_api_error_fail_on_error_false(): + """Test async_moderation_hook when API call fails and fail_on_error is False""" + mock_user_api_key_dict = UserAPIKeyAuth() + + guardrail = ModelArmorGuardrail( + template_id="test-template", + project_id="test-project", + location="us-central1", + guardrail_name="model-armor-test", + optional_params={"fail_on_error": False} + ) + + # Mock the access token method + guardrail._ensure_access_token_async = AsyncMock(return_value=("test-token", "test-project")) + + # Mock the async handler to raise an exception + guardrail.async_handler = AsyncMock() + guardrail.async_handler.post = AsyncMock(side_effect=Exception("API Error")) + + request_data = { + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "Hello, how are you?"} + ], + "metadata": {"guardrails": ["model-armor-test"]} + } + + # Even with fail_on_error=False, the decorator may still raise the exception + # This test verifies that the exception is properly logged and handled + with pytest.raises(Exception) as exc_info: + await guardrail.async_moderation_hook( + data=request_data, + user_api_key_dict=mock_user_api_key_dict, + call_type="completion" + ) + + assert "API Error" in str(exc_info.value) \ No newline at end of file From 4535b5847d3cd950b393e4ffb78ae6886feaa2f9 Mon Sep 17 00:00:00 2001 From: Shanto Mathew Date: Mon, 27 Oct 2025 15:35:52 -0500 Subject: [PATCH 04/91] docs(openrouter): add base_url config with environment variables (#15946) Added a new "Configuration with Environment Variables" section demonstrating: - Using os.getenv() to dynamically retrieve OpenRouter configuration - Explicitly passing base_url parameter with environment variables - Benefits of this approach for managing configs across environments This helps users implement production-ready configuration patterns. --- docs/my-website/docs/providers/openrouter.md | 33 +++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/providers/openrouter.md b/docs/my-website/docs/providers/openrouter.md index 58a87f68495..327634909b3 100644 --- a/docs/my-website/docs/providers/openrouter.md +++ b/docs/my-website/docs/providers/openrouter.md @@ -9,10 +9,9 @@ LiteLLM supports all the text / chat / vision models from [OpenRouter](https://o ```python import os from litellm import completion + os.environ["OPENROUTER_API_KEY"] = "" os.environ["OPENROUTER_API_BASE"] = "" # [OPTIONAL] defaults to https://openrouter.ai/api/v1 - - os.environ["OR_SITE_URL"] = "" # [OPTIONAL] os.environ["OR_APP_NAME"] = "" # [OPTIONAL] @@ -22,8 +21,32 @@ response = completion( ) ``` -## OpenRouter Completion Models +## Configuration with Environment Variables +For production environments, you can dynamically configure the base_url using environment variables: + +```python +import os +from litellm import completion + +# Configure with environment variables +OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY") +OPENROUTER_BASE_URL = os.getenv("OPENROUTER_API_BASE", "https://openrouter.ai/api/v1") + +# Set environment for LiteLLM +os.environ["OPENROUTER_API_KEY"] = OPENROUTER_API_KEY +os.environ["OPENROUTER_API_BASE"] = OPENROUTER_BASE_URL + +response = completion( + model="openrouter/google/palm-2-chat-bison", + messages=messages, + base_url=OPENROUTER_BASE_URL # Explicitly pass base_url for clarity +) +``` + +This approach provides better flexibility for managing configurations across different environments (dev, staging, production) and makes it easier to switch between self-hosted and cloud endpoints. + +## OpenRouter Completion Models 🚨 LiteLLM supports ALL OpenRouter models, send `model=openrouter/` to send it to open router. See all openrouter models [here](https://openrouter.ai/models) | Model Name | Function Call | @@ -40,12 +63,12 @@ response = completion( | openrouter/meta-llama/llama-2-70b-chat | `completion('openrouter/meta-llama/llama-2-70b-chat', messages)` | `os.environ['OR_SITE_URL']`,`os.environ['OR_APP_NAME']`,`os.environ['OPENROUTER_API_KEY']` | ## Passing OpenRouter Params - transforms, models, route - Pass `transforms`, `models`, `route`as arguments to `litellm.completion()` ```python import os from litellm import completion + os.environ["OPENROUTER_API_KEY"] = "" response = completion( @@ -54,4 +77,4 @@ response = completion( transforms = [""], route= "" ) -``` \ No newline at end of file +``` From 20f9e189fb2eaf21ce6dd782ae418c5a9455c6d4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 Oct 2025 13:44:55 -0700 Subject: [PATCH 05/91] [Buf fix] - Azure OpenAI, fix ContextWindowExceededError is not mapped from Azure openai errors (#15981) * fix is_error_str_context_window_exceeded * test_is_error_str_context_window_exceeded --- litellm/litellm_core_utils/exception_mapping_utils.py | 1 + .../litellm_core_utils/test_exception_mapping_utils.py | 1 + 2 files changed, 2 insertions(+) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index c6d3637ffcb..61551b04236 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -67,6 +67,7 @@ class ExceptionCheckers: "string too long. expected a string with maximum length", "model's maximum context limit", "is longer than the model's context length", + "input tokens exceed the configured limit", ] for substring in known_exception_substrings: if substring in _error_str_lowercase: diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py index 42e65612a96..638aab8fb49 100644 --- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py @@ -11,6 +11,7 @@ context_window_test_cases = [ ("Validation Error: string too long. expected a string with maximum length 1000.", True), ("Your prompt is longer than the model's context length of 2048.", True), ("AWS Bedrock Error: The request payload size has exceed context limit.", True), + ("Input tokens exceed the configured limit of 272000 tokens. Your messages resulted in 509178 tokens. Please reduce the length of the messages.", True), # Test case insensitivity ("ERROR: THIS MODEL'S MAXIMUM CONTEXT LENGTH IS 1024.", True), From 02df4c6b30ed262c31da82f0c67c149abc62ec22 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 Oct 2025 13:45:09 -0700 Subject: [PATCH 06/91] [Fix] DD logging - ensure key's metadata + guardrail is logged on DD (#15980) * fix get_sanitized_user_information_from_key * test_get_sanitized_user_information_from_key_includes_guardrails_metadata --- litellm/proxy/litellm_pre_call_utils.py | 3 ++- litellm/proxy/proxy_config.yaml | 3 +++ .../proxy/test_litellm_pre_call_utils.py | 21 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 3534695d9f8..1e53655dd0b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -585,7 +585,8 @@ class LiteLLMProxyRequestSetup: if user_api_key_dict.budget_reset_at else None ), - user_api_key_auth_metadata=None, + + user_api_key_auth_metadata=user_api_key_dict.metadata, ) return user_api_key_logged_metadata diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index c727d14dc99..f28af41c26a 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -21,3 +21,6 @@ search_tools: litellm_params: search_provider: exa_ai api_key: os.environ/EXA_API_KEY + +litellm_settings: + callbacks: ["datadog"] \ No newline at end of file diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index bb95bd4109a..bcc0e5ac2d6 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -1119,3 +1119,24 @@ def test_add_internal_user_from_user_mapping_no_header_or_mapping_returns_unchan ) assert result is user_api_key_dict assert user_api_key_dict.user_id is None + + +def test_get_sanitized_user_information_from_key_includes_guardrails_metadata(): + """ + Test that get_sanitized_user_information_from_key includes guardrails field from key metadata in the returned payload + """ + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key-hash", + key_alias="test-alias", + user_id="test-user", + metadata={"guardrails": ["presidio", "aporia"], "other_field": "value"}, + ) + + result = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key( + user_api_key_dict=user_api_key_dict + ) + + assert result["user_api_key_auth_metadata"] is not None + assert "guardrails" in result["user_api_key_auth_metadata"] + assert result["user_api_key_auth_metadata"]["guardrails"] == ["presidio", "aporia"] + assert result["user_api_key_auth_metadata"]["other_field"] == "value" From 17f6238d2b429ed36ac544e8a4a0f6348ef8f1ae Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 Oct 2025 13:45:21 -0700 Subject: [PATCH 07/91] [Feat] OTEL - Ensure error information is logged on OTEL (#15978) * fix _record_exception_on_span * _record_exception_on_span * test_record_exception_on_span * fix linting errors --- litellm/integrations/_types/open_inference.py | 39 +++++++ litellm/integrations/opentelemetry.py | 85 ++++++++++++++ .../unified_guardrail/unified_guardrail.py | 6 +- .../test_opentelemetry_unit_tests.py | 105 ++++++++++++++++++ 4 files changed, 231 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/_types/open_inference.py b/litellm/integrations/_types/open_inference.py index 65ecadcf370..af2ff2347c8 100644 --- a/litellm/integrations/_types/open_inference.py +++ b/litellm/integrations/_types/open_inference.py @@ -387,3 +387,42 @@ class OpenInferenceLLMProviderValues(Enum): GOOGLE = "google" AZURE = "azure" AWS = "aws" + + +class ErrorAttributes: + """ + Attributes for error information in spans. + + These attributes follow OpenTelemetry semantic conventions for exceptions + and are used to record error information from StandardLoggingPayloadErrorInformation. + """ + + ERROR_TYPE = "error.type" + """ + The type/class of the error (e.g., 'ValueError', 'OpenAIError', 'RateLimitError'). + Corresponds to StandardLoggingPayloadErrorInformation.error_class + """ + + ERROR_MESSAGE = "error.message" + """ + The error message describing what went wrong. + Corresponds to StandardLoggingPayloadErrorInformation.error_message + """ + + ERROR_CODE = "error.code" + """ + The error code (e.g., HTTP status code like '500', '429', or provider-specific codes). + Corresponds to StandardLoggingPayloadErrorInformation.error_code + """ + + ERROR_STACK_TRACE = "error.stack_trace" + """ + The full stack trace of the error. + Corresponds to StandardLoggingPayloadErrorInformation.traceback + """ + + ERROR_LLM_PROVIDER = "error.llm_provider" + """ + The LLM provider where the error occurred (e.g., 'openai', 'anthropic', 'azure'). + Corresponds to StandardLoggingPayloadErrorInformation.llm_provider + """ diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 11f2c6b2618..9315384ad96 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -841,6 +841,10 @@ class OpenTelemetry(CustomLogger): ) span.set_status(Status(StatusCode.ERROR)) self.set_attributes(span, kwargs, response_obj) + + # Record exception information using OTEL standard method + self._record_exception_on_span(span=span, kwargs=kwargs) + span.end(end_time=self._to_ns(end_time)) # Create span for guardrail information @@ -849,6 +853,87 @@ class OpenTelemetry(CustomLogger): if parent_otel_span is not None: parent_otel_span.end(end_time=self._to_ns(datetime.now())) + def _record_exception_on_span(self, span: Span, kwargs: dict): + """ + Record exception information on the span using OTEL standard methods. + + This extracts error information from StandardLoggingPayload and: + 1. Uses span.record_exception() for the actual exception object (OTEL standard) + 2. Sets structured error attributes from StandardLoggingPayloadErrorInformation + """ + try: + from litellm.integrations._types.open_inference import ErrorAttributes + + # Get the exception object if available + exception = kwargs.get("exception") + + # Record the exception using OTEL's standard method + if exception is not None: + span.record_exception(exception) + + # Get StandardLoggingPayload for structured error information + standard_logging_payload: Optional[StandardLoggingPayload] = kwargs.get( + "standard_logging_object" + ) + + if standard_logging_payload is None: + return + + # Extract error_information from StandardLoggingPayload + error_information = standard_logging_payload.get("error_information") + + if error_information is None: + # Fallback to error_str if error_information is not available + error_str = standard_logging_payload.get("error_str") + if error_str: + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_MESSAGE, + value=error_str, + ) + return + + # Set structured error attributes from StandardLoggingPayloadErrorInformation + if error_information.get("error_code"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_CODE, + value=error_information["error_code"], + ) + + if error_information.get("error_class"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_TYPE, + value=error_information["error_class"], + ) + + if error_information.get("error_message"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_MESSAGE, + value=error_information["error_message"], + ) + + if error_information.get("llm_provider"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_LLM_PROVIDER, + value=error_information["llm_provider"], + ) + + if error_information.get("traceback"): + self.safe_set_attribute( + span=span, + key=ErrorAttributes.ERROR_STACK_TRACE, + value=error_information["traceback"], + ) + + except Exception as e: + verbose_logger.exception( + "OpenTelemetry: Error recording exception on span: %s", str(e) + ) + def set_tools_attributes(self, span: Span, tools): import json diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 0bd1589ed1e..5355f308fd0 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -13,15 +13,13 @@ from litellm.caching.caching import DualCache from litellm.cost_calculator import _infer_call_type from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger -from litellm.llms import ( - endpoint_guardrail_translation_mappings, - load_guardrail_translation_mappings, -) +from litellm.llms import load_guardrail_translation_mappings from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks from litellm.types.utils import CallTypes, ModelResponseStream GUARDRAIL_NAME = "unified_llm_guardrails" +endpoint_guardrail_translation_mappings = None class UnifiedLLMGuardrails(CustomLogger): diff --git a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py index a1b80e0d1db..807ca24333e 100644 --- a/tests/logging_callback_tests/test_opentelemetry_unit_tests.py +++ b/tests/logging_callback_tests/test_opentelemetry_unit_tests.py @@ -119,3 +119,108 @@ class TestOpentelemetryUnitTests(BaseLoggingCallbackTest): assert detected_span_context.span_id == parent_span_context.span_id, ( "Detected span should have same span_id as parent" ) + + def test_record_exception_on_span(self): + """ + Test that _record_exception_on_span properly records exception information. + + This test verifies that StandardLoggingPayloadErrorInformation is properly + extracted and set as span attributes using ErrorAttributes constants. + """ + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from litellm.integrations.opentelemetry import OpenTelemetry + from litellm.integrations._types.open_inference import ErrorAttributes + + # Setup: Create TracerProvider and tracer + tracer_provider = TracerProvider() + trace.set_tracer_provider(tracer_provider) + tracer = trace.get_tracer(__name__) + + # Create OpenTelemetry integration + otel_integration = OpenTelemetry() + + # Create a mock span + mock_span = MagicMock() + + # Create test exception + test_exception = ValueError("Test error message") + + # Create kwargs with exception and error_information + kwargs = { + "exception": test_exception, + "standard_logging_object": { + "error_information": { + "error_code": "500", + "error_class": "ValueError", + "llm_provider": "openai", + "traceback": "Traceback (most recent call last)...", + "error_message": "Test error message", + }, + "error_str": "Test error message", + }, + } + + # Act: Record exception on span + otel_integration._record_exception_on_span(span=mock_span, kwargs=kwargs) + + # Assert: span.record_exception should be called with the exception + mock_span.record_exception.assert_called_once_with(test_exception) + + # Assert: Error attributes should be set using ErrorAttributes constants + expected_calls = [ + (ErrorAttributes.ERROR_CODE, "500"), + (ErrorAttributes.ERROR_TYPE, "ValueError"), + (ErrorAttributes.ERROR_MESSAGE, "Test error message"), + (ErrorAttributes.ERROR_LLM_PROVIDER, "openai"), + (ErrorAttributes.ERROR_STACK_TRACE, "Traceback (most recent call last)..."), + ] + + # Check that set_attribute was called with expected values + actual_calls = [call.args for call in mock_span.set_attribute.call_args_list] + + for expected_call in expected_calls: + assert expected_call in actual_calls, ( + f"Expected set_attribute call {expected_call} not found in actual calls: {actual_calls}" + ) + + def test_record_exception_on_span_with_fallback(self): + """ + Test that _record_exception_on_span falls back to error_str when error_information is None. + """ + from opentelemetry import trace + from opentelemetry.sdk.trace import TracerProvider + from litellm.integrations.opentelemetry import OpenTelemetry + from litellm.integrations._types.open_inference import ErrorAttributes + + # Setup: Create TracerProvider and tracer + tracer_provider = TracerProvider() + trace.set_tracer_provider(tracer_provider) + tracer = trace.get_tracer(__name__) + + # Create OpenTelemetry integration + otel_integration = OpenTelemetry() + + # Create a mock span + mock_span = MagicMock() + + # Create test exception + test_exception = ValueError("Test error message") + + # Create kwargs without error_information (should fallback to error_str) + kwargs = { + "exception": test_exception, + "standard_logging_object": { + "error_information": None, + "error_str": "Fallback error message", + }, + } + + # Act: Record exception on span + otel_integration._record_exception_on_span(span=mock_span, kwargs=kwargs) + + # Assert: span.record_exception should be called + mock_span.record_exception.assert_called_once_with(test_exception) + + # Assert: error.message should be set from error_str using ErrorAttributes constant + mock_span.set_attribute.assert_called_with(ErrorAttributes.ERROR_MESSAGE, "Fallback error message") From 65afdda3ef05f88570af9be86348422e7286df4c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 Oct 2025 13:45:34 -0700 Subject: [PATCH 08/91] fix exception triggered only when not logging (#15982) --- litellm/proxy/hooks/proxy_track_cost_callback.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 06f92261d89..e165f96b663 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -186,10 +186,6 @@ class _ProxyDBLogger(CustomLogger): response_cost=response_cost, max_budget=end_user_max_budget, ) - else: - raise Exception( - "User API key and team id and user id missing from custom callback." - ) else: if kwargs["stream"] is not True or ( kwargs["stream"] is True and "complete_streaming_response" in kwargs From 0bb53f504803bcf21eb72cd4357afb6f618c266a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 Oct 2025 13:45:44 -0700 Subject: [PATCH 09/91] [Fix] Azure OpenAI - Add handling for `v1` under azure api versions (#15984) * fix _is_azure_v1_api_version * test_is_azure_v1_api_version --- litellm/llms/azure/common_utils.py | 2 +- .../llms/azure/test_azure_common_utils.py | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index dfe662cc165..d9c5bea1a3f 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -759,4 +759,4 @@ class BaseAzureLLM(BaseOpenAILLM): def _is_azure_v1_api_version(api_version: Optional[str]) -> bool: if api_version is None: return False - return api_version == "preview" or api_version == "latest" + return api_version in {"preview", "latest", "v1"} diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index d4f6e3d8fb3..114ab3603d6 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -1504,3 +1504,23 @@ def test_get_azure_ad_token_fallback_to_default_azure_credential(setup_mocks, mo # Verify the token is what we expect from our DefaultAzureCredential mock assert token == "mock-default-azure-credential-token" + + +@pytest.mark.parametrize( + "api_version,expected", + [ + ("preview", True), + ("latest", True), + ("v1", True), + (None, False), + ("2023-05-15", False), + ("2024-01-01", False), + ("", False), + ], +) +def test_is_azure_v1_api_version(api_version, expected): + """ + Test that _is_azure_v1_api_version correctly identifies v1 API versions. + """ + result = BaseAzureLLM._is_azure_v1_api_version(api_version=api_version) + assert result == expected From 59df75276c970585cbbc654ac24936cd9b9f8b5c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 28 Oct 2025 02:16:21 +0530 Subject: [PATCH 10/91] Fix: Respect `LiteLLM-Disable-Message-Redaction` header for Responses API (#15966) * fix overide for logging unredacted messages * Use _get_metadata_variable_name_from_kwargs * fix test related to redaction --- litellm/litellm_core_utils/core_helpers.py | 16 ++++ litellm/litellm_core_utils/redact_messages.py | 15 ++-- litellm/router.py | 7 +- .../test_logging_redaction_e2e_test.py | 86 +++++++++++++++++++ 4 files changed, 117 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 7423e55b626..298935459b6 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -138,6 +138,22 @@ def add_missing_spend_metadata_to_litellm_metadata( return litellm_metadata +def get_metadata_variable_name_from_kwargs( + kwargs: dict, +) -> str: + """ + Helper to return what the "metadata" field should be called in the request data + + - New endpoints return `litellm_metadata` + - Old endpoints return `metadata` + + Context: + - LiteLLM used `metadata` as an internal field for storing metadata + - OpenAI then started using this field for their metadata + - LiteLLM is now moving to using `litellm_metadata` for our metadata + """ + return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" + def get_litellm_metadata_from_kwargs(kwargs: dict): """ Helper to get litellm metadata from all litellm request kwargs diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 5ac38949e2b..4237f005465 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -14,6 +14,9 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.secret_managers.main import str_to_bool from litellm.types.utils import StandardCallbackDynamicParams +from litellm.litellm_core_utils.core_helpers import ( + get_metadata_variable_name_from_kwargs, +) import asyncio if TYPE_CHECKING: @@ -107,11 +110,13 @@ def should_redact_message_logging(model_call_details: dict) -> bool: """ Determine if message logging should be redacted. """ - _request_headers = ( - model_call_details.get("litellm_params", {}).get("metadata", {}) or {} - ) - - request_headers = _request_headers.get("headers", {}) + litellm_params = model_call_details.get("litellm_params", {}) + + metadata_field = get_metadata_variable_name_from_kwargs(litellm_params) + metadata = litellm_params.get(metadata_field, {}) + + # Get headers from the metadata + request_headers = metadata.get("headers", {}) if isinstance(metadata, dict) else {} possible_request_headers = [ "litellm-enable-message-redaction", # old header. maintain backwards compatibility diff --git a/litellm/router.py b/litellm/router.py index 5b6fc56f292..d772bcdbe45 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -54,7 +54,10 @@ from litellm.caching.caching import ( from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.asyncify import run_async_function -from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs +from litellm.litellm_core_utils.core_helpers import ( + _get_parent_otel_span_from_kwargs, + get_metadata_variable_name_from_kwargs, +) from litellm.litellm_core_utils.coroutine_checker import coroutine_checker from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.dd_tracing import tracer @@ -4756,7 +4759,7 @@ class Router: - OpenAI then started using this field for their metadata - LiteLLM is now moving to using `litellm_metadata` for our metadata """ - return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" + return get_metadata_variable_name_from_kwargs(kwargs) def log_retry(self, kwargs: dict, e: Exception) -> dict: """ diff --git a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py index 227b57220e1..6b13c877ce9 100644 --- a/tests/logging_callback_tests/test_logging_redaction_e2e_test.py +++ b/tests/logging_callback_tests/test_logging_redaction_e2e_test.py @@ -279,3 +279,89 @@ async def test_redaction_with_streaming_response(): "logged standard logging payload for streaming with coroutine handling", json.dumps(standard_logging_payload, indent=2), ) + + +@pytest.mark.asyncio +async def test_disable_redaction_header_responses_api(): + """ + Test that LiteLLM-Disable-Message-Redaction header works for Responses API. + + This test verifies the fix for the issue where the header wasn't respected + because Responses API uses 'litellm_metadata' instead of 'metadata'. + """ + litellm.turn_off_message_logging = True + test_custom_logger = TestCustomLogger() + litellm.callbacks = [test_custom_logger] + + # Mock a ResponsesAPIResponse-style response + mock_response = { + "output": [{"text": "This is a test response"}], + "model": "gpt-3.5-turbo", + "usage": {"input_tokens": 5, "output_tokens": 5, "total_tokens": 10} + } + + # Pass the header via litellm_metadata (as the proxy does for Responses API) + response = await litellm.aresponses( + model="gpt-3.5-turbo", + input="hi", + mock_response=mock_response, + litellm_metadata={ + "headers": { + "litellm-disable-message-redaction": "true" + } + } + ) + + await asyncio.sleep(1) + standard_logging_payload = test_custom_logger.logged_standard_logging_payload + assert standard_logging_payload is not None + + # Verify that messages are NOT redacted because the header was set + print( + "logged standard logging payload for ResponsesAPI with disable header", + json.dumps(standard_logging_payload, indent=2, default=str), + ) + + # The content should NOT be redacted + assert standard_logging_payload["response"] != {"text": "redacted-by-litellm"} + assert standard_logging_payload["messages"][0]["content"] == "hi" + + +@pytest.mark.asyncio +async def test_redaction_with_metadata_completion_api(): + """ + Test redaction behavior with metadata field for Completion API. + + This test verifies that get_metadata_variable_name_from_kwargs properly + selects the appropriate metadata field for header detection. + """ + litellm.turn_off_message_logging = True + test_custom_logger = TestCustomLogger() + litellm.callbacks = [test_custom_logger] + + # When metadata is passed, the system uses get_metadata_variable_name_from_kwargs + # to determine which field to check + response = await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "hi"}], + mock_response="hello", + metadata={ + "headers": { + "litellm-disable-message-redaction": "true" + } + } + ) + + await asyncio.sleep(1) + standard_logging_payload = test_custom_logger.logged_standard_logging_payload + assert standard_logging_payload is not None + + print( + "logged standard logging payload for Completion API with metadata", + json.dumps(standard_logging_payload, indent=2), + ) + + # Verify the helper function works correctly - with get_metadata_variable_name_from_kwargs, + # the system checks the appropriate field for headers + assert standard_logging_payload["response"] == {"text": "redacted-by-litellm"} + assert standard_logging_payload["messages"][0]["content"] == "redacted-by-litellm" From cb57455172f9dc184cc5bd82489175e87cea251b Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 27 Oct 2025 13:48:23 -0700 Subject: [PATCH 11/91] test_foward_litellm_user_info_to_backend_llm_call --- tests/proxy_unit_tests/test_proxy_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 55ec5ee32a1..38c0bf934d9 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -530,6 +530,7 @@ def test_foward_litellm_user_info_to_backend_llm_call(): "x-litellm-user_api_key_org_id": "test_org_id", "x-litellm-user_api_key_hash": "test_api_key", "x-litellm-user_api_key_spend": 0.0, + "x-litellm-user_api_key_auth_metadata": {}, } assert json.dumps(data, sort_keys=True) == json.dumps(expected_data, sort_keys=True) From 2d836dfb6d1aabd4a87156f381de9ca069880ccf Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 27 Oct 2025 13:49:47 -0700 Subject: [PATCH 12/91] test_basic_moderations_on_proxy_with_model --- tests/otel_tests/test_moderations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/otel_tests/test_moderations.py b/tests/otel_tests/test_moderations.py index a0218b039fa..822e2558889 100644 --- a/tests/otel_tests/test_moderations.py +++ b/tests/otel_tests/test_moderations.py @@ -58,7 +58,7 @@ async def test_basic_moderations_on_proxy_with_model(): test_text = "I want to harm someone" # Test text that should trigger moderation request_data = { "input": test_text, - "model": "text-moderation-stable", + "model": "omni-moderation-latest", } try: response = await make_moderations_curl_request( From 1acc321eb3560899703d780486fd11732ba747c9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 27 Oct 2025 13:50:32 -0700 Subject: [PATCH 13/91] test_router_amoderation --- tests/local_testing/test_router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index e6032f21722..55d5c418c3d 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -1700,7 +1700,7 @@ async def test_router_amoderation(): { "model_name": "openai-moderations", "litellm_params": { - "model": "text-moderation-stable", + "model": "omni-moderation-latest", "api_key": os.getenv("OPENAI_API_KEY", None), }, } @@ -1709,7 +1709,7 @@ async def test_router_amoderation(): router = Router(model_list=model_list) ## Test 1: user facing function result = await router.amoderation( - model="text-moderation-stable", input="this is valid good text" + model="omni-moderation-latest", input="this is valid good text" ) From 4ed9c7d7f2d0b72e99abca7e1325e1e55693c86b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 27 Oct 2025 15:46:04 -0700 Subject: [PATCH 14/91] [Feat] UI - Changed API Base from Select to Input in New LLM Credentials (#15987) * Changed API Base from Select to Input * Added Tests --- ui/litellm-dashboard/package-lock.json | 80 +++++++++++++++++++ ui/litellm-dashboard/package.json | 1 + .../provider_specific_fields.test.tsx | 47 +++++++++++ .../add_model/provider_specific_fields.tsx | 16 ++-- 4 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 5f59711ab50..7205738a21c 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -39,6 +39,7 @@ }, "devDependencies": { "@tailwindcss/forms": "^0.5.7", + "@testing-library/dom": "^10.4.1", "@testing-library/jest-dom": "^6.8.0", "@testing-library/react": "^16.3.0", "@testing-library/user-event": "^14.6.1", @@ -5356,6 +5357,26 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, + "node_modules/@testing-library/dom": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", + "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.10.4", + "@babel/runtime": "^7.12.5", + "@types/aria-query": "^5.0.1", + "aria-query": "5.3.0", + "dom-accessibility-api": "^0.5.9", + "lz-string": "^1.5.0", + "picocolors": "1.1.1", + "pretty-format": "^27.0.2" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@testing-library/jest-dom": { "version": "6.8.0", "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.8.0.tgz", @@ -5462,6 +5483,13 @@ "node": ">=10.13.0" } }, + "node_modules/@types/aria-query": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", + "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -9823,6 +9851,13 @@ "node": ">=6.0.0" } }, + "node_modules/dom-accessibility-api": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", + "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", + "dev": true, + "license": "MIT" + }, "node_modules/dom-converter": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz", @@ -14016,6 +14051,16 @@ "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/lz-string": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz", + "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", + "dev": true, + "license": "MIT", + "bin": { + "lz-string": "bin/bin.js" + } + }, "node_modules/magic-string": { "version": "0.30.19", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.19.tgz", @@ -17956,6 +18001,41 @@ "renderkid": "^3.0.0" } }, + "node_modules/pretty-format": { + "version": "27.5.1", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", + "integrity": "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1", + "ansi-styles": "^5.0.0", + "react-is": "^17.0.1" + }, + "engines": { + "node": "^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/pretty-format/node_modules/react-is": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", + "integrity": "sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w==", + "dev": true, + "license": "MIT" + }, "node_modules/pretty-time": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/pretty-time/-/pretty-time-1.1.0.tgz", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 4746b893545..102ea91ae17 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -44,6 +44,7 @@ }, "devDependencies": { "@tailwindcss/forms": "^0.5.7", + "@testing-library/dom": "^10.4.1", "@testing-library/jest-dom": "^6.8.0", "@testing-library/react": "^16.3.0", "@testing-library/user-event": "^14.6.1", diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx new file mode 100644 index 00000000000..4475cd4ca3f --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.test.tsx @@ -0,0 +1,47 @@ +import { render, waitFor } from "@testing-library/react"; +import { describe, it, expect, beforeAll } from "vitest"; +import { Form } from "antd"; +import { Providers } from "../provider_info_helpers"; +import ProviderSpecificFields from "./provider_specific_fields"; + +// Mock window.matchMedia for Ant Design components +beforeAll(() => { + Object.defineProperty(window, "matchMedia", { + writable: true, + value: (query: string) => ({ + matches: false, + media: query, + onchange: null, + addListener: () => {}, // deprecated + removeListener: () => {}, // deprecated + addEventListener: () => {}, + removeEventListener: () => {}, + dispatchEvent: () => false, + }), + }); +}); + +describe("ProviderSpecificFields", () => { + it("should render the provider specific fields for OpenAI", async () => { + const { getByLabelText, getByPlaceholderText } = render( +
+ + , + ); + + await waitFor(() => { + // Check for the API Base text input + const apiBaseInput = getByPlaceholderText("https://api.openai.com/v1"); + expect(apiBaseInput).toBeInTheDocument(); + expect(apiBaseInput).toHaveAttribute("type", "text"); + + // Check for Organization field + const orgInput = getByPlaceholderText("[OPTIONAL] my-unique-org"); + expect(orgInput).toBeInTheDocument(); + + // Check for API Key field + const apiKeyLabel = getByLabelText("OpenAI API Key"); + expect(apiKeyLabel).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx index 2d11569ab95..e42f1bfba6a 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx @@ -68,8 +68,9 @@ const PROVIDER_CREDENTIAL_FIELDS: Record = { key: "api_base", label: "API Base", - type: "select", - options: ["https://api.openai.com/v1", "https://eu.api.openai.com"], + type: "text", + placeholder: "https://api.openai.com/v1", + tooltip: "Common endpoints: https://api.openai.com/v1, https://eu.api.openai.com, https://us.api.openai.com", defaultValue: "https://api.openai.com/v1", }, { @@ -88,8 +89,9 @@ const PROVIDER_CREDENTIAL_FIELDS: Record = { key: "api_base", label: "API Base", - type: "select", - options: ["https://api.openai.com/v1", "https://eu.api.openai.com"], + type: "text", + placeholder: "https://api.openai.com/v1", + tooltip: "Common endpoints: https://api.openai.com/v1, https://eu.api.openai.com, https://us.api.openai.com", defaultValue: "https://api.openai.com/v1", }, { @@ -651,7 +653,11 @@ const ProviderSpecificFields: React.FC = ({ selecte }>Click to Upload ) : ( - + )} From 64167b7e34e85f8ece89b46104835e03e362d9d6 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 27 Oct 2025 17:20:58 -0700 Subject: [PATCH 15/91] Remove limit from admin UI numerical input fix (#15991) --- .../KeyInfoView.handleKeyUpdate.test.tsx | 22 +++++++++++++++++++ .../components/templates/key_info_view.tsx | 8 +++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx b/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx index 4b5ed6ae2f9..8257cd62880 100644 --- a/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx +++ b/ui/litellm-dashboard/src/components/templates/KeyInfoView.handleKeyUpdate.test.tsx @@ -332,3 +332,25 @@ describe("KeyInfoView handleKeyUpdate premium guard", () => { expect(sentPayload.key).toBe("tok_123"); }); }); + +describe("KeyInfoView handleKeyUpdate empty strings", () => { + ["tpm_limit", "rpm_limit", "max_parallel_requests", "max_budget"].forEach((limit) => { + it(`maps empty strings to null for ${limit}`, async () => { + renderView(true); // premiumUser = true + + fireEvent.click(screen.getByText("Edit Settings")); + (globalThis as any).__TEST_FORM_VALUES = { + token: "tok_123", + [limit]: "", + }; + + fireEvent.click(screen.getByText("Mock Submit")); + + await waitFor(() => expect(keyUpdateCallMock).toHaveBeenCalled()); + + const [sentAccessToken, sentPayload] = keyUpdateCallMock.mock.calls[0]; + expect(sentAccessToken).toBe("access_abc"); + expect(sentPayload[limit]).toBeNull(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index 425b863ff79..9e0c95900f8 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -144,10 +144,10 @@ export default function KeyInfoView({ delete formValues.mcp_tool_permissions; } - // Handle max_budget empty string - if (formValues.max_budget === "") { - formValues.max_budget = null; - } + formValues.max_budget = mapEmptyStringToNull(formValues.max_budget); + formValues.tpm_limit = mapEmptyStringToNull(formValues.tpm_limit); + formValues.rpm_limit = mapEmptyStringToNull(formValues.rpm_limit); + formValues.max_parallel_requests = mapEmptyStringToNull(formValues.max_parallel_requests); // Convert metadata back to an object if it exists and is a string if (formValues.metadata && typeof formValues.metadata === "string") { From de6fffe743629119ab4c28adab402afed4c93aed Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 27 Oct 2025 17:22:54 -0700 Subject: [PATCH 16/91] =?UTF-8?q?bump:=20version=201.79.0=20=E2=86=92=201.?= =?UTF-8?q?79.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6bcf5dcf3a9..2fedd1d3bd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.79.0" +version = "1.79.1" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -157,7 +157,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.79.0" +version = "1.79.1" version_files = [ "pyproject.toml:^version" ] From d5f48c7e23ae5930810e5e1bd9108c7d6053d52e Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 27 Oct 2025 17:38:24 -0700 Subject: [PATCH 17/91] get_metadata_variable_name_from_kwargs --- litellm/litellm_core_utils/core_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 298935459b6..8b9f53cec15 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -1,6 +1,6 @@ # What is this? ## Helper utilities -from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union +from typing import TYPE_CHECKING, Any, Iterable, List, Literal, Optional, Union import httpx @@ -140,7 +140,7 @@ def add_missing_spend_metadata_to_litellm_metadata( def get_metadata_variable_name_from_kwargs( kwargs: dict, -) -> str: +) -> Literal["metadata", "litellm_metadata"]: """ Helper to return what the "metadata" field should be called in the request data From 0e23f89eb776bd39395ba772bf9789984853ba46 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 27 Oct 2025 17:47:19 -0700 Subject: [PATCH 18/91] fix ModelArmorGuardrail --- .../guardrails/guardrail_hooks/model_armor/model_armor.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py index 47c6cb7e01a..ab524aec89c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py +++ b/litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py @@ -56,6 +56,14 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase): api_endpoint: Optional[str] = None, **kwargs, ): + # Set supported event hooks if not already provided + if "event_hook" not in kwargs: + kwargs["event_hook"] = [ + GuardrailEventHooks.pre_call, + GuardrailEventHooks.during_call, + GuardrailEventHooks.post_call, + ] + # Initialize parent classes first super().__init__(**kwargs) VertexBase.__init__(self) From a3d64fb843415e50fb498c4ba1d784a80df296f7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Mon, 27 Oct 2025 17:48:35 -0700 Subject: [PATCH 19/91] fix omni-moderation-latest --- tests/local_testing/test_openai_moderations_hook.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/local_testing/test_openai_moderations_hook.py b/tests/local_testing/test_openai_moderations_hook.py index 2ab86699515..c63ce342b25 100644 --- a/tests/local_testing/test_openai_moderations_hook.py +++ b/tests/local_testing/test_openai_moderations_hook.py @@ -34,7 +34,7 @@ async def test_openai_moderation_error_raising(): """ openai_mod = _ENTERPRISE_OpenAI_Moderation() - litellm.openai_moderations_model_name = "text-moderation-latest" + litellm.openai_moderations_model_name = "omni-moderation-latest" _api_key = "sk-12345" _api_key = hash_token("sk-12345") user_api_key_dict = UserAPIKeyAuth(api_key=_api_key) @@ -45,9 +45,9 @@ async def test_openai_moderation_error_raising(): llm_router = litellm.Router( model_list=[ { - "model_name": "text-moderation-latest", + "model_name": "omni-moderation-latest", "litellm_params": { - "model": "text-moderation-latest", + "model": "omni-moderation-latest", "api_key": os.environ["OPENAI_API_KEY"], }, } From 43af45ab88f53bd9a07b3f2b7793333600071706 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 27 Oct 2025 18:00:38 -0700 Subject: [PATCH 20/91] Key Already Exist Error Notification (#15993) --- .../molecules/notifications_manager.test.tsx | 41 +++++++++++++++++++ .../molecules/notifications_manager.tsx | 3 +- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 ui/litellm-dashboard/src/components/molecules/notifications_manager.test.tsx diff --git a/ui/litellm-dashboard/src/components/molecules/notifications_manager.test.tsx b/ui/litellm-dashboard/src/components/molecules/notifications_manager.test.tsx new file mode 100644 index 00000000000..c57c3052170 --- /dev/null +++ b/ui/litellm-dashboard/src/components/molecules/notifications_manager.test.tsx @@ -0,0 +1,41 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { notification } from "antd"; +import NotificationManager from "./notifications_manager"; + +// Mock the antd notification module +vi.mock("antd", () => ({ + notification: { + error: vi.fn(), + warning: vi.fn(), + info: vi.fn(), + success: vi.fn(), + destroy: vi.fn(), + }, +})); + +describe("NotificationManager", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe("Already Exists case", () => { + it("should show error notification for 'already exists' message", () => { + const error = { + message: "Key with alias 'test10' already exists.", + type: "bad_request_error", + code: "400", + }; + + NotificationManager.fromBackend(error); + + expect(notification.error).toHaveBeenCalledWith( + expect.objectContaining({ + message: "Already Exists", + description: "Key with alias 'test10' already exists.", + duration: 6, + placement: "topRight", + }), + ); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/molecules/notifications_manager.tsx b/ui/litellm-dashboard/src/components/molecules/notifications_manager.tsx index 559fa2e03c8..6c6a0e38f21 100644 --- a/ui/litellm-dashboard/src/components/molecules/notifications_manager.tsx +++ b/ui/litellm-dashboard/src/components/molecules/notifications_manager.tsx @@ -317,7 +317,8 @@ const NotificationManager = { title === "Authentication Error" || title === "Access Denied" || title === "Not Found" || - title === "Error" + title === "Error" || + title === "Already Exists" ) { notification.error({ ...payload, duration: extra?.duration ?? 6 }); return; From 4cef208c5f2b42fd879935f502bff33aca88ce6d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 27 Oct 2025 19:12:13 -0700 Subject: [PATCH 21/91] [Fix] - Responses API - add /openai routes for responses API. (Azure OpenAI SDK Compatibility) (#15988) * add /openai routes for responses API * TestResponsesAPIEndpoints --- .../proxy/response_api_endpoints/endpoints.py | 25 +++++++++ .../proxy/response_api_endpoints/__init__.py | 0 .../response_api_endpoints/test_endpoints.py | 53 +++++++++++++++++++ 3 files changed, 78 insertions(+) create mode 100644 tests/test_litellm/proxy/response_api_endpoints/__init__.py create mode 100644 tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index c87690854f3..26d10c1ac47 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -17,6 +17,11 @@ router = APIRouter() dependencies=[Depends(user_api_key_auth)], tags=["responses"], ) +@router.post( + "/openai/v1/responses", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) async def responses_api( request: Request, fastapi_response: Response, @@ -90,6 +95,11 @@ async def responses_api( dependencies=[Depends(user_api_key_auth)], tags=["responses"], ) +@router.get( + "/openai/v1/responses/{response_id}", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) async def get_response( response_id: str, request: Request, @@ -162,6 +172,11 @@ async def get_response( dependencies=[Depends(user_api_key_auth)], tags=["responses"], ) +@router.delete( + "/openai/v1/responses/{response_id}", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) async def delete_response( response_id: str, request: Request, @@ -234,6 +249,11 @@ async def delete_response( dependencies=[Depends(user_api_key_auth)], tags=["responses"], ) +@router.get( + "/openai/v1/responses/{response_id}/input_items", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) async def get_response_input_items( response_id: str, request: Request, @@ -297,6 +317,11 @@ async def get_response_input_items( dependencies=[Depends(user_api_key_auth)], tags=["responses"], ) +@router.post( + "/openai/v1/responses/{response_id}/cancel", + dependencies=[Depends(user_api_key_auth)], + tags=["responses"], +) async def cancel_response( response_id: str, request: Request, diff --git a/tests/test_litellm/proxy/response_api_endpoints/__init__.py b/tests/test_litellm/proxy/response_api_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py new file mode 100644 index 00000000000..bca0944aeac --- /dev/null +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -0,0 +1,53 @@ +""" +Test for response_api_endpoints/endpoints.py +""" +import unittest +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from litellm.proxy.proxy_server import app + + +class TestResponsesAPIEndpoints(unittest.TestCase): + @pytest.mark.asyncio + @patch("litellm.proxy.proxy_server.llm_router") + @patch("litellm.proxy.proxy_server.user_api_key_auth") + async def test_openai_v1_responses_route(self, mock_auth, mock_router): + """ + Test that /openai/v1/responses endpoint is correctly registered and accessible. + """ + mock_auth.return_value = MagicMock( + token="test_token", + user_id="test_user", + team_id=None, + ) + + mock_router.aresponses = AsyncMock( + return_value={ + "id": "resp_abc123", + "object": "realtime.response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "Test response"}], + } + ], + } + ) + + client = TestClient(app) + + test_data = {"model": "gpt-4o", "input": "Tell me about AI"} + + response = client.post( + "/openai/v1/responses", + json=test_data, + headers={"Authorization": "Bearer sk-1234"}, + ) + + assert response.status_code in [200, 401, 500] + From c5c37bf7f545452c60340c2562464227ef826c3f Mon Sep 17 00:00:00 2001 From: dima-hx430 Date: Tue, 28 Oct 2025 04:39:38 +0200 Subject: [PATCH 22/91] Add models missing deprecation dates (#15976) --- model_prices_and_context_window.json | 30 +++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2eced0d9095..eac5ec6461c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1106,6 +1106,7 @@ "supports_vision": true }, "azure/eu/gpt-4o-2024-08-06": { + "deprecation_date": "2026-02-27", "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, "litellm_provider": "azure", @@ -1122,6 +1123,7 @@ "supports_vision": true }, "azure/eu/gpt-4o-2024-11-20": { + "deprecation_date": "2026-03-01", "cache_creation_input_token_cost": 1.38e-06, "input_cost_per_token": 2.75e-06, "litellm_provider": "azure", @@ -1280,7 +1282,7 @@ }, "azure/global-standard/gpt-4o-2024-08-06": { "cache_read_input_token_cost": 1.25e-06, - "deprecation_date": "2025-08-20", + "deprecation_date": "2026-02-27", "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -1297,7 +1299,7 @@ }, "azure/global-standard/gpt-4o-2024-11-20": { "cache_read_input_token_cost": 1.25e-06, - "deprecation_date": "2025-12-20", + "deprecation_date": "2026-03-01", "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -1326,6 +1328,7 @@ "supports_vision": true }, "azure/global/gpt-4o-2024-08-06": { + "deprecation_date": "2026-02-27", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -1342,6 +1345,7 @@ "supports_vision": true }, "azure/global/gpt-4o-2024-11-20": { + "deprecation_date": "2026-03-01", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -1625,6 +1629,7 @@ "supports_web_search": false }, "azure/gpt-4.1-2025-04-14": { + "deprecation_date": "2026-11-04", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -1691,6 +1696,7 @@ "supports_web_search": false }, "azure/gpt-4.1-mini-2025-04-14": { + "deprecation_date": "2026-11-04", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, @@ -1756,6 +1762,7 @@ "supports_vision": true }, "azure/gpt-4.1-nano-2025-04-14": { + "deprecation_date": "2026-11-04", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -1837,6 +1844,7 @@ "supports_vision": true }, "azure/gpt-4o-2024-08-06": { + "deprecation_date": "2026-02-27", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -1853,6 +1861,7 @@ "supports_vision": true }, "azure/gpt-4o-2024-11-20": { + "deprecation_date": "2026-03-01", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.75e-06, "litellm_provider": "azure", @@ -2604,6 +2613,7 @@ "supports_vision": true }, "azure/o3-2025-04-16": { + "deprecation_date": "2026-04-16", "cache_read_input_token_cost": 2.5e-06, "input_cost_per_token": 1e-05, "litellm_provider": "azure", @@ -2832,6 +2842,7 @@ "output_cost_per_token": 0.0 }, "azure/text-embedding-3-small": { + "deprecation_date": "2026-04-30", "input_cost_per_token": 2e-08, "litellm_provider": "azure", "max_input_tokens": 8191, @@ -2870,6 +2881,7 @@ "mode": "audio_speech" }, "azure/us/gpt-4o-2024-08-06": { + "deprecation_date": "2026-02-27", "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, "litellm_provider": "azure", @@ -2886,6 +2898,7 @@ "supports_vision": true }, "azure/us/gpt-4o-2024-11-20": { + "deprecation_date": "2026-03-01", "cache_creation_input_token_cost": 1.38e-06, "input_cost_per_token": 2.75e-06, "litellm_provider": "azure", @@ -4911,7 +4924,7 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-02-01", + "deprecation_date": "2026-02-19", "input_cost_per_token": 3e-06, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -4968,7 +4981,6 @@ "cache_creation_input_token_cost": 3e-07, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2025-03-01", "input_cost_per_token": 2.5e-07, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -4988,7 +5000,7 @@ "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 1.5e-06, - "deprecation_date": "2025-03-01", + "deprecation_date": "2026-05-01", "input_cost_per_token": 1.5e-05, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -5172,6 +5184,7 @@ "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, + "deprecation_date": "2026-08-05", "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 32000, @@ -5199,6 +5212,7 @@ "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, + "deprecation_date": "2026-05-14", "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 32000, @@ -5222,6 +5236,7 @@ "tool_use_system_prompt_tokens": 159 }, "claude-sonnet-4-20250514": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -7993,6 +8008,7 @@ "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, + "deprecation_date": "2026-10-15", "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, @@ -12397,6 +12413,7 @@ "supports_tool_choice": true }, "gpt-3.5-turbo-1106": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 1e-06, "litellm_provider": "openai", "max_input_tokens": 16385, @@ -12466,6 +12483,7 @@ "supports_tool_choice": true }, "gpt-4-0125-preview": { + "deprecation_date": "2026-03-26", "input_cost_per_token": 1e-05, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -12506,6 +12524,7 @@ "supports_tool_choice": true }, "gpt-4-1106-preview": { + "deprecation_date": "2026-03-26", "input_cost_per_token": 1e-05, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -16563,6 +16582,7 @@ "supports_vision": true }, "o1-mini-2024-09-12": { + "deprecation_date": "2025-10-27", "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 3e-06, "litellm_provider": "openai", From 5ad108bc9b16c451dcc3e0cd5079d62d7b69ae1e Mon Sep 17 00:00:00 2001 From: Mac Misiura <82826099+m-misiura@users.noreply.github.com> Date: Tue, 28 Oct 2025 02:41:10 +0000 Subject: [PATCH 23/91] :memo: updated `ibm_guardrails.md` to better indicate how detectors could be configured (#15971) --- .../docs/proxy/guardrails/ibm_guardrails.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/proxy/guardrails/ibm_guardrails.md b/docs/my-website/docs/proxy/guardrails/ibm_guardrails.md index 6d1f2eae911..0c13d2dcea9 100644 --- a/docs/my-website/docs/proxy/guardrails/ibm_guardrails.md +++ b/docs/my-website/docs/proxy/guardrails/ibm_guardrails.md @@ -3,11 +3,18 @@ import TabItem from '@theme/TabItem'; # IBM Guardrails -LiteLLM works with IBM's FMS Guardrails for content safety. You can use it to detect jailbreaks, PII, hate speech, and more. +LiteLLM works with [IBM's FMS Guardrails](https://github.com/foundation-model-stack/fms-guardrails-orchestrator) for content safety. You can use it to detect jailbreaks, PII, hate speech, and more. ## What it does -IBM Guardrails analyzes text and tells you if it contains things you want to avoid. It gives each detection a score. Higher scores mean it's more confident. +IBM's FMS Guardrails is a framework for invoking detectors on LLM inputs and outputs. To configure these detectors, you can use e.g. [TrustyAI detectors](https://github.com/trustyai-explainability/guardrails-detectors), an open-source project maintained by the Red Hat's [TrustyAI team](https://github.com/trustyai-explainability) that allows the user to configure detectors that are: + +- regex patterns +- file type validators +- custom Python functions +- Hugging Face [AutoModelForSequenceClassification](https://huggingface.co/docs/transformers/en/model_doc/auto#transformers.AutoModelForSequenceClassification), i.e. sequence classification models + +Each detector outputs an API response based on the following [openapi schema](https://foundation-model-stack.github.io/fms-guardrails-orchestrator/docs/api/openapi_detector_api.yaml). You can run these checks: - Before sending to the LLM (on user input) @@ -73,7 +80,7 @@ curl -i http://localhost:4000/v1/chat/completions \ - `guardrail` - str - Set to `ibm_guardrails` - `auth_token` - str - Your IBM Guardrails auth token. Can use `os.environ/IBM_GUARDRAILS_AUTH_TOKEN` -- `base_url` - str - URL of your IBM Guardrails server +- `base_url` - str - URL of your IBM Detector or Guardrails server - `detector_id` - str - Which detector to use (e.g., "jailbreak-detector", "pii-detector") ### Optional params @@ -97,7 +104,7 @@ IBM Guardrails has two APIs you can use: ### Detector Server (recommended) -The simpler one. Sends all messages at once. +[This Detectors API](https://foundation-model-stack.github.io/fms-guardrails-orchestrator/?urls.primaryName=Detector+API#/Text) uses `api/v1/text/contents` endpoint to run a single detector; it can accept multiple text inputs within a request. ```yaml guardrails: @@ -113,7 +120,7 @@ guardrails: ### Orchestrator -If you're using the IBM FMS Guardrails Orchestrator, you can use this. +If you're using the IBM FMS Guardrails Orchestrator, you can use [FMS Orchestrator API](https://foundation-model-stack.github.io/fms-guardrails-orchestrator/?urls.primaryName=Orchestrator+API), specifically by leveraging the `api/v2/text/detection/content` to potentially run multiple detectors in a single request; however, this endpoint can only accept one text input per request. ```yaml guardrails: From 8b33328cc12e1bd41b5f092b934b866679949166 Mon Sep 17 00:00:00 2001 From: YutaSaito <36355491+uc4w6c@users.noreply.github.com> Date: Tue, 28 Oct 2025 11:43:40 +0900 Subject: [PATCH 24/91] Perf speed up pytest (#15951) * perf: Skip sleep delays in base_mail.py during tests to improve test speed * perf: Mock datetime.now in parallel_request_limiter_v3.py to improve test speed * pref: Mock urllib system calls in test_aiohttp_transport.py to improve test speed * chore: add --durations=50 to visualize slowest tests * pref: reduce setup phase overhead by widening fixture scope in conftest.py * test: stabilize flaky tests * fix: minor issue --- .github/workflows/test-litellm.yml | 3 +- .../proxy/hooks/dynamic_rate_limiter_v3.py | 13 ++++-- .../hooks/parallel_request_limiter_v3.py | 17 +++++-- tests/test_litellm/conftest.py | 2 +- .../send_emails/test_base_email.py | 11 +++-- .../custom_httpx/test_aiohttp_transport.py | 2 + .../test_vertex_gemma_transformation.py | 7 +++ .../hooks/test_dynamic_rate_limiter_v3.py | 28 ++++++++++-- .../hooks/test_parallel_request_limiter_v3.py | 44 ++++++++++++++----- .../proxy/management_endpoints/test_ui_sso.py | 18 +++----- .../test_spend_management_endpoints.py | 8 ++++ 11 files changed, 116 insertions(+), 37 deletions(-) diff --git a/.github/workflows/test-litellm.yml b/.github/workflows/test-litellm.yml index b7f4a25d593..1d9bd201fa8 100644 --- a/.github/workflows/test-litellm.yml +++ b/.github/workflows/test-litellm.yml @@ -33,6 +33,7 @@ jobs: poetry run pip install "google-genai==1.22.0" poetry run pip install "google-cloud-aiplatform>=1.38" poetry run pip install "fastapi-offline==1.7.3" + poetry run pip install "python-multipart==0.0.18" - name: Setup litellm-enterprise as local package run: | cd enterprise @@ -40,4 +41,4 @@ jobs: cd .. - name: Run tests run: | - poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 + poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50 diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index d266aadc7b3..5bf2c0aba61 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -3,7 +3,8 @@ Dynamic rate limiter v3 - Saturation-aware priority-based rate limiting """ import os -from typing import Dict, List, Literal, Optional, Union +from datetime import datetime +from typing import Callable, Dict, List, Literal, Optional, Union from fastapi import HTTPException @@ -42,9 +43,15 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): - When saturated: strict priority-based limits enforced (fair) - Uses v3 limiter's atomic Lua scripts for race-free increments """ - def __init__(self, internal_usage_cache: DualCache): + def __init__( + self, + internal_usage_cache: DualCache, + time_provider: Optional[Callable[[], datetime]] = None, + ): self.internal_usage_cache = InternalUsageCache(dual_cache=internal_usage_cache) - self.v3_limiter = _PROXY_MaxParallelRequestsHandler_v3(self.internal_usage_cache) + self.v3_limiter = _PROXY_MaxParallelRequestsHandler_v3( + self.internal_usage_cache, time_provider=time_provider + ) def update_variables(self, llm_router: Router): self.llm_router = llm_router diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 15090187381..dfaa8ccdd84 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -11,6 +11,7 @@ from math import floor from typing import ( TYPE_CHECKING, Any, + Callable, Dict, List, Literal, @@ -137,8 +138,13 @@ class RateLimitResponseWithDescriptors(TypedDict): class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): - def __init__(self, internal_usage_cache: InternalUsageCache): + def __init__( + self, + internal_usage_cache: InternalUsageCache, + time_provider: Optional[Callable[[], datetime]] = None, + ): self.internal_usage_cache = internal_usage_cache + self._time_provider = time_provider or datetime.now if self.internal_usage_cache.dual_cache.redis_cache is not None: self.batch_rate_limiter_script = ( self.internal_usage_cache.dual_cache.redis_cache.async_register_script( @@ -156,6 +162,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): self.window_size = int(os.getenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", 60)) + def _get_current_time(self) -> datetime: + """Return the current time for rate limiting calculations.""" + return self._time_provider() + def _is_redis_cluster(self) -> bool: """ Check if the dual cache is using Redis cluster. @@ -425,7 +435,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): read_only: If True, only check limits without incrementing counters """ - now = datetime.now().timestamp() + current_time = self._get_current_time() + now = current_time.timestamp() now_int = int(now) # Convert to integer for Redis Lua script # Collect all keys and their metadata upfront @@ -1090,7 +1101,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): descriptor = descriptors[floor(i / 2)] # Calculate reset time (window_start + window_size) - now = datetime.now().timestamp() + now = self._get_current_time().timestamp() reset_time = now + self.window_size # Conservative estimate reset_time_formatted = datetime.fromtimestamp( reset_time diff --git a/tests/test_litellm/conftest.py b/tests/test_litellm/conftest.py index ac8a00d850c..365ddfe03df 100644 --- a/tests/test_litellm/conftest.py +++ b/tests/test_litellm/conftest.py @@ -26,7 +26,7 @@ def event_loop(): -@pytest.fixture(scope="function", autouse=True) +@pytest.fixture(scope="module", autouse=True) def setup_and_teardown(): """ This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py index 4fac979ce65..d9886bd7100 100644 --- a/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py +++ b/tests/test_litellm/enterprise/enterprise_callbacks/send_emails/test_base_email.py @@ -4,13 +4,11 @@ import sys import unittest.mock as mock from unittest.mock import patch +from enterprise.litellm_enterprise.enterprise_callbacks.send_emails.base_email import BaseEmailLogger import pytest from fastapi.testclient import TestClient sys.path.insert(0, os.path.abspath("../../..")) -from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( - BaseEmailLogger, -) from litellm_enterprise.types.enterprise_callbacks.send_emails import ( EmailEvent, SendKeyCreatedEmailEvent, @@ -20,6 +18,13 @@ from litellm.integrations.email_templates.email_footer import EMAIL_FOOTER from litellm.proxy._types import Litellm_EntityType, WebhookEvent +@pytest.fixture(autouse=True) +def no_invitation_wait(monkeypatch): + async def _noop(self): + return None + + monkeypatch.setattr(BaseEmailLogger, "_wait_for_invitation_creation", _noop) + @pytest.fixture def base_email_logger(): return BaseEmailLogger() diff --git a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py index 4059b5b2e74..894fc45e361 100644 --- a/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py +++ b/tests/test_litellm/llms/custom_httpx/test_aiohttp_transport.py @@ -188,6 +188,8 @@ async def test_handle_async_request_uses_env_proxy(monkeypatch): monkeypatch.setenv("HTTPS_PROXY", proxy_url) monkeypatch.setenv("https_proxy", proxy_url) monkeypatch.delenv("DISABLE_AIOHTTP_TRUST_ENV", raising=False) + monkeypatch.setattr("urllib.request.getproxies", lambda: {"http": proxy_url, "https": proxy_url}) + monkeypatch.setattr("urllib.request.proxy_bypass", lambda host: False) captured = {} diff --git a/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py index bfa86647373..eab010ddfda 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py @@ -11,6 +11,13 @@ import pytest import litellm +@pytest.fixture(autouse=True) +def _reset_litellm_http_client_cache(): + """Ensure each test gets a fresh async HTTP client mock.""" + from litellm import in_memory_llm_clients_cache + + in_memory_llm_clients_cache.flush_cache() + class TestVertexGemmaCompletion: """Test completion flow for Vertex AI Gemma models using litellm.acompletion()""" diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index d3cbd460cf0..1f76013e237 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -8,6 +8,7 @@ import asyncio import os import sys import time +from datetime import datetime, timedelta from unittest.mock import AsyncMock, patch import pytest @@ -22,6 +23,24 @@ from litellm.proxy.hooks.dynamic_rate_limiter_v3 import ( ) +class TimeController: + def __init__(self): + self._current = datetime.utcnow() + + def now(self) -> datetime: + return self._current + + def advance(self, seconds: float) -> None: + self._current += timedelta(seconds=seconds) + + +@pytest.fixture +def time_controller(monkeypatch): + controller = TimeController() + monkeypatch.setattr(time, "time", lambda: controller.now().timestamp()) + return controller + + @pytest.mark.asyncio async def test_priority_weight_allocation(): """ @@ -195,7 +214,7 @@ async def test_concurrent_priority_requests(): @pytest.mark.asyncio -async def test_100_concurrent_priority_requests(): +async def test_100_concurrent_priority_requests(time_controller): """ Stress test: 100 concurrent requests with mixed priorities over 10 seconds. @@ -211,7 +230,9 @@ async def test_100_concurrent_priority_requests(): litellm.priority_reservation = {"high": 0.9, "low": 0.1} dual_cache = DualCache() - handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache) + handler = DynamicRateLimitHandler( + internal_usage_cache=dual_cache, time_provider=time_controller.now + ) model = "stress-test-model" total_tpm = 1000 @@ -307,7 +328,8 @@ async def test_100_concurrent_priority_requests(): # Add small delay between batches to spread over ~10 seconds if batch_idx < len(batches) - 1: # Don't sleep after last batch - await asyncio.sleep(1.0) # 1 second between batches + await asyncio.sleep(0) + time_controller.advance(1.0) # simulate 1s passing between batches end_time = time.time() total_duration = end_time - start_time diff --git a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py index a56dd8632d4..87f40383b7e 100644 --- a/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py @@ -5,7 +5,8 @@ Unit Tests for the max parallel request limiter v3 for the proxy import asyncio import os import sys -from datetime import datetime +import time +from datetime import datetime, timedelta from typing import Any, Dict, List, Optional import pytest @@ -21,10 +22,27 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( from litellm.proxy.utils import InternalUsageCache, ProxyLogging, hash_token from litellm.types.utils import ModelResponse, Usage +class TimeController: + def __init__(self): + self._current = datetime.utcnow() + + def now(self) -> datetime: + return self._current + + def advance(self, seconds: float) -> None: + self._current += timedelta(seconds=seconds) + + +@pytest.fixture +def time_controller(monkeypatch): + controller = TimeController() + monkeypatch.setattr(time, "time", lambda: controller.now().timestamp()) + return controller + @pytest.mark.flaky(reruns=3) @pytest.mark.asyncio -async def test_sliding_window_rate_limit_v3(monkeypatch): +async def test_sliding_window_rate_limit_v3(monkeypatch, time_controller): """ Test the sliding window rate limiting functionality """ @@ -34,7 +52,8 @@ async def test_sliding_window_rate_limit_v3(monkeypatch): user_api_key_dict = UserAPIKeyAuth(api_key=_api_key, rpm_limit=3) local_cache = DualCache() parallel_request_handler = _PROXY_MaxParallelRequestsHandler( - internal_usage_cache=InternalUsageCache(local_cache) + internal_usage_cache=InternalUsageCache(local_cache), + time_provider=time_controller.now, ) # Mock the batch_rate_limiter_script to simulate window expiry and use correct key construction @@ -103,7 +122,7 @@ async def test_sliding_window_rate_limit_v3(monkeypatch): assert "Rate limit exceeded" in str(exc_info.value.detail) # Wait for window to expire (2 seconds) - await asyncio.sleep(3) + time_controller.advance(3) print("WAITED 3 seconds") @@ -116,7 +135,7 @@ async def test_sliding_window_rate_limit_v3(monkeypatch): @pytest.mark.asyncio -async def test_rate_limiter_script_return_values_v3(monkeypatch): +async def test_rate_limiter_script_return_values_v3(monkeypatch, time_controller): """ Test that the rate limiter script returns both counter and window values correctly """ @@ -126,7 +145,8 @@ async def test_rate_limiter_script_return_values_v3(monkeypatch): user_api_key_dict = UserAPIKeyAuth(api_key=_api_key, rpm_limit=3) local_cache = DualCache() parallel_request_handler = _PROXY_MaxParallelRequestsHandler( - internal_usage_cache=InternalUsageCache(local_cache) + internal_usage_cache=InternalUsageCache(local_cache), + time_provider=time_controller.now, ) # Mock the batch_rate_limiter_script to simulate window expiry and use correct key construction @@ -199,7 +219,7 @@ async def test_rate_limiter_script_return_values_v3(monkeypatch): assert new_counter_value == 2, "Counter should be 2 after second request" # Wait for window to expire - await asyncio.sleep(3) + time_controller.advance(3) # Make request after window expiry await parallel_request_handler.async_pre_call_hook( @@ -226,7 +246,7 @@ async def test_rate_limiter_script_return_values_v3(monkeypatch): ) @pytest.mark.flaky(reruns=3) @pytest.mark.asyncio -async def test_normal_router_call_tpm_v3(monkeypatch, rate_limit_object): +async def test_normal_router_call_tpm_v3(monkeypatch, rate_limit_object, time_controller): """ Test normal router call with parallel request limiter v3 for TPM rate limiting """ @@ -276,7 +296,8 @@ async def test_normal_router_call_tpm_v3(monkeypatch, rate_limit_object): ) local_cache = DualCache() parallel_request_handler = _PROXY_MaxParallelRequestsHandler( - internal_usage_cache=InternalUsageCache(local_cache) + internal_usage_cache=InternalUsageCache(local_cache), + time_provider=time_controller.now, ) # Mock the batch_rate_limiter_script to simulate window expiry and use correct key construction @@ -359,7 +380,8 @@ async def test_normal_router_call_tpm_v3(monkeypatch, rate_limit_object): }, mock_response="hello", ) - await asyncio.sleep(1) # success is done in a separate thread + await asyncio.sleep(0) + time_controller.advance(1) # Verify the token count is tracked counter_value = await local_cache.async_get_cache(key=counter_key) @@ -383,7 +405,7 @@ async def test_normal_router_call_tpm_v3(monkeypatch, rate_limit_object): ) # Wait for window to expire - await asyncio.sleep(3) + time_controller.advance(3) # Make request after window expiry await parallel_request_handler.async_pre_call_hook( diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 0f403ae5d65..42cec94322c 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -125,16 +125,14 @@ def test_get_microsoft_callback_response(): "surname": "User", } - future = asyncio.Future() - future.set_result(mock_response) - with patch.dict( os.environ, {"MICROSOFT_CLIENT_SECRET": "mock_secret", "MICROSOFT_TENANT": "mock_tenant"}, ): + mock_verify = AsyncMock(return_value=mock_response) with patch( "fastapi_sso.sso.microsoft.MicrosoftSSO.verify_and_process", - return_value=future, + new=mock_verify, ): # Act result = asyncio.run( @@ -166,15 +164,14 @@ def test_get_microsoft_callback_response_raw_sso_response(): "surname": "User", } - future = asyncio.Future() - future.set_result(mock_response) with patch.dict( os.environ, {"MICROSOFT_CLIENT_SECRET": "mock_secret", "MICROSOFT_TENANT": "mock_tenant"}, ): + mock_verify = AsyncMock(return_value=mock_response) with patch( "fastapi_sso.sso.microsoft.MicrosoftSSO.verify_and_process", - return_value=future, + new=mock_verify, ): # Act result = asyncio.run( @@ -207,12 +204,10 @@ def test_get_google_callback_response(): "family_name": "User", } - future = asyncio.Future() - future.set_result(mock_response) - with patch.dict(os.environ, {"GOOGLE_CLIENT_SECRET": "mock_secret"}): + mock_verify = AsyncMock(return_value=mock_response) with patch( - "fastapi_sso.sso.google.GoogleSSO.verify_and_process", return_value=future + "fastapi_sso.sso.google.GoogleSSO.verify_and_process", new=mock_verify ): # Act result = asyncio.run( @@ -2072,4 +2067,3 @@ class TestPKCEFunctionality: assert "code_challenge=" in updated_location assert "code_challenge_method=S256" in updated_location assert f"state={test_state}" in updated_location - diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 662064a1afc..4c82fb85bcd 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -71,6 +71,14 @@ def disable_budget_sync(monkeypatch): ) +@pytest.fixture(autouse=True) +def reset_router_callbacks(): + """Ensure router budget callbacks from previous tests do not leak state.""" + litellm.logging_callback_manager._reset_all_callbacks() + yield + litellm.logging_callback_manager._reset_all_callbacks() + + @pytest.mark.asyncio async def test_ui_view_spend_logs_with_user_id(client, monkeypatch): # Mock data for the test From 2bef7c3662dc0c5d5a03bfd1eb7698cb1c8bbe72 Mon Sep 17 00:00:00 2001 From: Chris Gibbons <83524181+ylgibby@users.noreply.github.com> Date: Mon, 27 Oct 2025 20:44:45 -0600 Subject: [PATCH 25/91] fix: Preserve Bedrock inference profile IDs in health checks (#15947) * fix: Preserve Bedrock inference profile IDs in health checks - Fixes issue where health checks were stripping inference profile IDs - Preserves cross-region inference profile prefixes (us., eu., apac., jp., au., us-gov., global.) - Strips only AWS region routing while preserving routes and handlers - Resolves both issue #15807 and inference profile requirement errors - Adds comprehensive tests for all Bedrock model format combinations Issue #15807 attempted to fix regional Bedrock model health checks but was too aggressive, stripping cross-region inference profile prefixes that AWS requires. This caused errors: "Invocation of model ID X with on-demand throughput isn't supported. Retry your request with the ID or ARN of an inference profile." The fix now correctly: - Strips AWS regions (us-west-2, eu-central-1, etc.) from routing - Preserves CRIS prefixes (us., eu., etc.) required by AWS - Preserves routes (converse/, invoke/) - Preserves handlers (llama/, deepseek_r1/) - Only affects Bedrock models (checked via startswith) Test coverage includes 20+ scenarios for all Bedrock model format combinations. * Remove unused traceback import --- litellm/proxy/health_check.py | 39 ++++++- .../litellm_utils_tests/test_health_check.py | 110 +++++++++++++++++- 2 files changed, 144 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index a0483a08bde..2f4bb081819 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -138,7 +138,7 @@ def _update_litellm_params_for_health_check( - gets a short `messages` param for health check - updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes - updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models - - updates the `model` param with the Bedrock base model name if it is a Bedrock model + - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID """ litellm_params["messages"] = _get_random_llm_message() _health_check_model = model_info.get("health_check_model", None) @@ -146,9 +146,42 @@ def _update_litellm_params_for_health_check( litellm_params["model"] = _health_check_model if model_info.get("mode", None) == "audio_speech": litellm_params["voice"] = model_info.get("health_check_voice", "alloy") - if "bedrock" in litellm_params["model"]: + + # Handle Bedrock region routing format: bedrock/region/model + # This is needed because health checks bypass get_llm_provider() for the model param + # Issue #15807: Without this, health checks send "region/model" as the model ID to AWS + # which causes: "bedrock-runtime.../model/us-west-2/mistral.../invoke" (region in model ID) + # + # However, we must preserve cross-region inference profile prefixes like "us.", "eu.", etc. + # Issue: Stripping these breaks AWS requirement for inference profile IDs + # + # Must also preserve route prefixes (converse/, invoke/) and handlers (llama/, deepseek_r1/, etc.) + if litellm_params["model"].startswith("bedrock/"): from litellm.llms.bedrock.common_utils import BedrockModelInfo - litellm_params["model"] = BedrockModelInfo.get_base_model(litellm_params["model"]) + + model = litellm_params["model"] + # Strip only the bedrock/ prefix (preserve routes like converse/, invoke/) + if model.startswith("bedrock/"): + model = model[8:] # len("bedrock/") = 8 + + # Now check for region routing and strip it if present + # Need to handle formats like: + # - "us-west-2/model" → "model" + # - "converse/us-west-2/model" → "converse/model" + # - "llama/arn:..." → "llama/arn:..." (preserve handler) + # + # Strategy: Check each path segment, remove regions, preserve everything else + parts = model.split("/") + filtered_parts = [] + + for part in parts: + # Skip AWS regions, keep everything else + if part not in BedrockModelInfo.all_global_regions: + filtered_parts.append(part) + + model = "/".join(filtered_parts) + litellm_params["model"] = model + return litellm_params diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py index 4236d0195d9..4d55a25f334 100644 --- a/tests/litellm_utils_tests/test_health_check.py +++ b/tests/litellm_utils_tests/test_health_check.py @@ -3,7 +3,6 @@ import os import sys -import traceback import pytest from unittest.mock import AsyncMock, patch @@ -302,7 +301,8 @@ def test_update_litellm_params_for_health_check(): updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) assert "voice" not in updated_params - # Test with Bedrock model + # Test with Bedrock model with region routing - should strip bedrock/ and region/ prefix + # Issue #15807: Fixes health checks sending "region/model" as model ID to AWS model_info = {} litellm_params = { "model": "bedrock/us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0", @@ -311,6 +311,112 @@ def test_update_litellm_params_for_health_check(): updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) assert updated_params["model"] == "anthropic.claude-3-7-sonnet-20250219-v1:0" + # Test with Bedrock cross-region inference profile - should preserve the inference profile prefix + # AWS requires inference profile IDs like "us.anthropic.claude..." for cross-region routing + litellm_params = { + "model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == "us.anthropic.claude-3-5-sonnet-20240620-v1:0" + + # Test with Bedrock model without region routing - should just strip bedrock/ prefix + litellm_params = { + "model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == "anthropic.claude-3-5-sonnet-20240620-v1:0" + + # Test that non-Bedrock models are not affected by Bedrock-specific logic + litellm_params = { + "model": "openai/gpt-4", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == "openai/gpt-4" # Should remain unchanged + + # Test ALL cross-region inference profile prefixes (CRIS) + cris_prefixes = ["us.", "eu.", "apac.", "jp.", "au.", "us-gov.", "global."] + for prefix in cris_prefixes: + litellm_params = { + "model": f"bedrock/{prefix}anthropic.claude-3-haiku-20240307-v1:0", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == f"{prefix}anthropic.claude-3-haiku-20240307-v1:0", \ + f"Failed to preserve CRIS prefix: {prefix}" + + # Test regional + CRIS combination - region should be stripped, CRIS preserved + litellm_params = { + "model": "bedrock/us-east-2/us.anthropic.claude-3-haiku-20240307-v1:0", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == "us.anthropic.claude-3-haiku-20240307-v1:0" + + # Test GovCloud regions + litellm_params = { + "model": "bedrock/us-gov-east-1/anthropic.claude-instant-v1", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == "anthropic.claude-instant-v1" + + # Test imported models with handler prefixes - handlers should be preserved + litellm_params = { + "model": "bedrock/llama/arn:aws:bedrock:us-east-1:123:imported-model/abc", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == "llama/arn:aws:bedrock:us-east-1:123:imported-model/abc" + + litellm_params = { + "model": "bedrock/deepseek_r1/arn:aws:bedrock:us-west-2:456:imported-model/xyz", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == "deepseek_r1/arn:aws:bedrock:us-west-2:456:imported-model/xyz" + + # Test route specifications - routes should be preserved + litellm_params = { + "model": "bedrock/converse/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == "converse/us.anthropic.claude-3-5-sonnet-20240620-v1:0" + + litellm_params = { + "model": "bedrock/invoke/us-west-2/anthropic.claude-instant-v1", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == "invoke/anthropic.claude-instant-v1" + + # Test ARN formats - should be preserved + litellm_params = { + "model": "bedrock/arn:aws:bedrock:eu-central-1:000:application-inference-profile/abc", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == "arn:aws:bedrock:eu-central-1:000:application-inference-profile/abc" + + # Test edge case: region + handler + ARN + litellm_params = { + "model": "bedrock/us-west-2/llama/arn:aws:bedrock:us-east-1:123:imported-model/abc", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == "llama/arn:aws:bedrock:us-east-1:123:imported-model/abc" + + # Test edge case: route + region + CRIS + litellm_params = { + "model": "bedrock/converse/us-west-2/eu.anthropic.claude-3-sonnet-20240229-v1:0", + "api_key": "fake_key", + } + updated_params = _update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == "converse/eu.anthropic.claude-3-sonnet-20240229-v1:0" + @pytest.mark.asyncio async def test_perform_health_check_with_health_check_model(): """ From 2074b4d6629629e5edc7147ce7c7528cf28c34e4 Mon Sep 17 00:00:00 2001 From: Katsuhiro Muto <63308909+eycjur@users.noreply.github.com> Date: Tue, 28 Oct 2025 11:47:31 +0900 Subject: [PATCH 26/91] Fix: Support tool usage messages with Langfuse OTEL integration (#15932) * Log tool use in langfuse otel integration * Add test for logging function calling --------- Co-authored-by: eycjur --- .../integrations/langfuse/langfuse_otel.py | 77 +++++++++++- litellm/types/integrations/langfuse_otel.py | 4 + .../integrations/test_langfuse_otel.py | 113 +++++++++++++++++- 3 files changed, 185 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py index fbe480be95f..43d16b5e4cb 100644 --- a/litellm/integrations/langfuse/langfuse_otel.py +++ b/litellm/integrations/langfuse/langfuse_otel.py @@ -48,11 +48,12 @@ class LangfuseOtelLogger(OpenTelemetry): _utils.set_attributes(span, kwargs, response_obj) ######################################################### - # Set Langfuse specific attributes eg Langfuse Environment + # Set Langfuse specific attributes ######################################################### LangfuseOtelLogger._set_langfuse_specific_attributes( span=span, - kwargs=kwargs + kwargs=kwargs, + response_obj=response_obj ) return @@ -86,7 +87,7 @@ class LangfuseOtelLogger(OpenTelemetry): return metadata @staticmethod - def _set_langfuse_specific_attributes(span: Span, kwargs): + def _set_langfuse_specific_attributes(span: Span, kwargs, response_obj): """ Sets Langfuse specific metadata attributes onto the OTEL span. @@ -96,6 +97,7 @@ class LangfuseOtelLogger(OpenTelemetry): compatibility. """ from litellm.integrations.arize._utils import safe_set_attribute + from litellm.litellm_core_utils.safe_json_dumps import safe_dumps # 1) Environment variable override langfuse_environment = os.environ.get("LANGFUSE_TRACING_ENVIRONMENT") @@ -141,6 +143,75 @@ class LangfuseOtelLogger(OpenTelemetry): value = str(value) safe_set_attribute(span, enum_attr.value, value) + # 3) Set observation input/output for better UI display + # + # These Langfuse-specific attributes provide better UI display, + # especially for tool calls and function calling. + # Set observation input (messages) + messages = kwargs.get("messages") + if messages: + safe_set_attribute( + span, + LangfuseSpanAttributes.OBSERVATION_INPUT.value, + safe_dumps(messages), + ) + + # Set observation output (response with tool_calls if present) + if response_obj and hasattr(response_obj, "get"): + choices = response_obj.get("choices", []) + if choices: + # Extract the first choice's message + first_choice = choices[0] + message = first_choice.get("message", {}) + + # Check if there are tool_calls + tool_calls = message.get("tool_calls") + if tool_calls: + # Transform tool_calls to Langfuse-expected format + transformed_tool_calls = [] + for tool_call in tool_calls: + function = tool_call.get("function", {}) + arguments_str = function.get("arguments", "{}") + + # Parse arguments from JSON string to object + try: + arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str + except json.JSONDecodeError: + arguments_obj = {} + + # Create Langfuse-compatible tool call object + langfuse_tool_call = { + "id": response_obj.get("id", ""), + "name": function.get("name", ""), + "call_id": tool_call.get("id", ""), + "type": "function_call", + "arguments": arguments_obj, + } + transformed_tool_calls.append(langfuse_tool_call) + + # Set the observation output with transformed tool_calls + safe_set_attribute( + span, + LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, + safe_dumps(transformed_tool_calls), + ) + else: + # No tool_calls, use regular content-based output + output_data = {} + + if message.get("role"): + output_data["role"] = message.get("role") + + if message.get("content") is not None: + output_data["content"] = message.get("content") + + if output_data: + safe_set_attribute( + span, + LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, + safe_dumps(output_data), + ) + @staticmethod def _get_langfuse_otel_host() -> Optional[str]: """ diff --git a/litellm/types/integrations/langfuse_otel.py b/litellm/types/integrations/langfuse_otel.py index 64fc036f45f..53d84c40052 100644 --- a/litellm/types/integrations/langfuse_otel.py +++ b/litellm/types/integrations/langfuse_otel.py @@ -24,6 +24,10 @@ class LangfuseSpanAttributes(str, Enum): MASK_INPUT = "langfuse.generation.mask_input" MASK_OUTPUT = "langfuse.generation.mask_output" + # ---- Observation input/output ---- + OBSERVATION_INPUT = "langfuse.observation.input" + OBSERVATION_OUTPUT = "langfuse.observation.output" + # ---- Trace-level metadata ---- TRACE_USER_ID = "user.id" SESSION_ID = "session.id" diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py index 59b14178320..a2fbbf74cec 100644 --- a/tests/test_litellm/integrations/test_langfuse_otel.py +++ b/tests/test_litellm/integrations/test_langfuse_otel.py @@ -92,7 +92,7 @@ class TestLangfuseOtelIntegration: with patch.dict(os.environ, {'LANGFUSE_TRACING_ENVIRONMENT': test_env}): with patch('litellm.integrations.arize._utils.safe_set_attribute') as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(mock_span, mock_kwargs) + LangfuseOtelLogger._set_langfuse_specific_attributes(mock_span, mock_kwargs, {}) # safe_set_attribute(span, key, value) → positional args mock_safe_set_attribute.assert_called_once_with( @@ -130,7 +130,7 @@ class TestLangfuseOtelIntegration: assert extracted.get("foo") == "bar" assert extracted.get("enriched") is True - def test_set_langfuse_specific_attributes_full_mapping(self): + def test_set_langfuse_specific_attributes_metadata(self): """Verify every supported metadata key maps to the correct OTEL attribute and complex types are JSON-serialised.""" # Build a sample metadata payload covering all mappings metadata = { @@ -156,7 +156,7 @@ class TestLangfuseOtelIntegration: # Capture calls to safe_set_attribute with patch('litellm.integrations.arize._utils.safe_set_attribute') as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs) + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, None) # Build expected calls manually for clarity from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes @@ -189,6 +189,109 @@ class TestLangfuseOtelIntegration: assert actual == expected, "Mismatch between expected and actual OTEL attribute mapping." + def test_set_langfuse_specific_attributes_with_content(self): + """Test that _set_langfuse_specific_attributes correctly sets observation.output with regular content response.""" + from litellm.types.utils import Choices, ModelResponse + from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes + + # Create response with content + response_obj = ModelResponse( + id='chatcmpl-test', + model='gpt-4o', + choices=[ + Choices( + finish_reason='stop', + message={ + "role": "assistant", + "content": "The weather in Tokyo is sunny." + } + ) + ], + ) + + kwargs = { + "messages": [{"role": "user", "content": "What's the weather in Tokyo?"}], + } + + with patch('litellm.integrations.arize._utils.safe_set_attribute') as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), kwargs, response_obj) + + expect_output = { + LangfuseSpanAttributes.OBSERVATION_INPUT.value: [ + { + "role": "user", + "content": "What's the weather in Tokyo?" + } + ], + LangfuseSpanAttributes.OBSERVATION_OUTPUT.value: { + "role": "assistant", + "content": "The weather in Tokyo is sunny." + } + } + + # Flatten the actual calls into {key: value} + actual = { + call.args[1]: json.loads(call.args[2]) + for call in mock_safe_set_attribute.call_args_list + } + + assert actual == expect_output, "Mismatch in observation input/output OTEL attributes." + + + def test_set_langfuse_specific_attributes_with_tool_calls(self): + """Test that _set_langfuse_specific_attributes correctly sets observation.output with tool calls in Langfuse format.""" + from litellm.types.utils import Choices, Function, ChatCompletionMessageToolCall, ModelResponse + from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes + + # Create response with tool calls + response_obj = ModelResponse( + id='chatcmpl-test', + model='gpt-4o', + choices=[ + Choices( + finish_reason='tool_calls', + message={ + "role": "assistant", + "content": None, + "tool_calls": [ + ChatCompletionMessageToolCall( + function=Function( + arguments='{"location":"Tokyo"}', + name='get_weather' + ), + id='call_123', + type='function' + ) + ] + } + ) + ], + ) + + with patch('litellm.integrations.arize._utils.safe_set_attribute') as mock_safe_set_attribute: + LangfuseOtelLogger._set_langfuse_specific_attributes(MagicMock(), {}, + response_obj) + + expected = { + LangfuseSpanAttributes.OBSERVATION_OUTPUT.value: [ + { + "id": "chatcmpl-test", + "name": "get_weather", + "arguments": {"location": "Tokyo"}, + "call_id": "call_123", + "type": "function_call" + } + ] + } + + # Flatten the actual calls into {key: value} + actual = { + call.args[1]: json.loads(call.args[2]) + for call in mock_safe_set_attribute.call_args_list + } + assert actual == expected, "Mismatch in observation output OTEL attribute for tool calls." + + def test_construct_dynamic_otel_headers_with_langfuse_keys(self): """Test that construct_dynamic_otel_headers creates proper auth headers when langfuse keys are provided.""" from litellm.types.utils import StandardCallbackDynamicParams @@ -352,7 +455,7 @@ class TestLangfuseOtelResponsesAPI: mock_span = MagicMock() with patch('litellm.integrations.arize._utils.safe_set_attribute') as mock_safe_set_attribute: - LangfuseOtelLogger._set_langfuse_specific_attributes(mock_span, kwargs) + LangfuseOtelLogger._set_langfuse_specific_attributes(mock_span, kwargs, {}) # Verify specific attributes were set from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes @@ -371,8 +474,6 @@ class TestLangfuseOtelResponsesAPI: for expected_call in expected_calls: mock_safe_set_attribute.assert_any_call(*expected_call) - - if __name__ == "__main__": pytest.main([__file__]) \ No newline at end of file From 2e7dc56895079b3e2747856e61c15307a7d655ea Mon Sep 17 00:00:00 2001 From: Thomas Schmidt Date: Tue, 28 Oct 2025 03:47:50 +0100 Subject: [PATCH 27/91] Add Haiku 4.5 pricing for open router (#15909) * Add Haiku 4.5 pricing for open router * Add haiku 4.5 pricing for open router --- ...odel_prices_and_context_window_backup.json | 30 +++++++++---------- model_prices_and_context_window.json | 30 +++++++++---------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 2eced0d9095..8297cc42ff9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -3385,7 +3385,7 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", "supports_reasoning": true, "supports_tool_choice": true - }, + }, "azure_ai/cohere-rerank-v3-english": { "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, @@ -7753,7 +7753,7 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true - }, + }, "dolphin": { "input_cost_per_token": 5e-07, "litellm_provider": "nlp_cloud", @@ -7847,7 +7847,7 @@ "input_cost_per_query": 5e-03, "litellm_provider": "perplexity", "mode": "search" - }, + }, "elevenlabs/scribe_v1": { "input_cost_per_second": 6.11e-05, "litellm_provider": "elevenlabs", @@ -17733,24 +17733,24 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-sonnet-4.5": { - "input_cost_per_image": 0.0048, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, + "openrouter/anthropic/claude-haiku-4.5": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, "litellm_provider": "openrouter", - "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "max_tokens": 1000000, + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "max_tokens": 200000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "tool_use_system_prompt_tokens": 346 }, "openrouter/bytedance/ui-tars-1.5-7b": { "input_cost_per_token": 1e-07, @@ -19166,7 +19166,7 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true - }, + }, "qwen.qwen3-32b-v1:0": { "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", @@ -19178,7 +19178,7 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true - }, + }, "recraft/recraftv2": { "litellm_provider": "recraft", "mode": "image_generation", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index eac5ec6461c..0623bc8a904 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3398,7 +3398,7 @@ "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/", "supports_reasoning": true, "supports_tool_choice": true - }, + }, "azure_ai/cohere-rerank-v3-english": { "input_cost_per_query": 0.002, "input_cost_per_token": 0.0, @@ -7768,7 +7768,7 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true - }, + }, "dolphin": { "input_cost_per_token": 5e-07, "litellm_provider": "nlp_cloud", @@ -7862,7 +7862,7 @@ "input_cost_per_query": 5e-03, "litellm_provider": "perplexity", "mode": "search" - }, + }, "elevenlabs/scribe_v1": { "input_cost_per_second": 6.11e-05, "litellm_provider": "elevenlabs", @@ -17753,24 +17753,24 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "openrouter/anthropic/claude-sonnet-4.5": { - "input_cost_per_image": 0.0048, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "output_cost_per_token_above_200k_tokens": 2.25e-05, + "openrouter/anthropic/claude-haiku-4.5": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, "litellm_provider": "openrouter", - "max_input_tokens": 1000000, - "max_output_tokens": 1000000, - "max_tokens": 1000000, + "max_input_tokens": 200000, + "max_output_tokens": 200000, + "max_tokens": 200000, "mode": "chat", - "output_cost_per_token": 1.5e-05, + "output_cost_per_token": 5e-06, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, + "supports_prompt_caching": true, "supports_reasoning": true, "supports_tool_choice": true, "supports_vision": true, - "tool_use_system_prompt_tokens": 159 + "tool_use_system_prompt_tokens": 346 }, "openrouter/bytedance/ui-tars-1.5-7b": { "input_cost_per_token": 1e-07, @@ -19186,7 +19186,7 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true - }, + }, "qwen.qwen3-32b-v1:0": { "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", @@ -19198,7 +19198,7 @@ "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true - }, + }, "recraft/recraftv2": { "litellm_provider": "recraft", "mode": "image_generation", From e27bab3238c14623808699658d6d48c625912855 Mon Sep 17 00:00:00 2001 From: Thomas Mildner <12685945+Thomas-Mildner@users.noreply.github.com> Date: Tue, 28 Oct 2025 03:48:40 +0100 Subject: [PATCH 28/91] fix(opik): enhance requester metadata retrieval from API key auth (#15897) --- litellm/integrations/opik/opik.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/litellm/integrations/opik/opik.py b/litellm/integrations/opik/opik.py index 9fa3482f663..c28aa14a11e 100644 --- a/litellm/integrations/opik/opik.py +++ b/litellm/integrations/opik/opik.py @@ -204,6 +204,13 @@ class OpikLogger(CustomBatchLogger): # Update litellm_opik_metadata with opik metadata from requester standard_logging_metadata = standard_logging_object.get("metadata", {}) or {} requester_metadata = standard_logging_metadata.get("requester_metadata", {}) or {} + + # If requester_metadata is empty, try to get it from user_api_key_auth_metadata saved in api key + if not requester_metadata: + requester_metadata = standard_logging_metadata.get( + "user_api_key_auth_metadata", {} + ) or {} + requester_opik_metadata = requester_metadata.get("opik", {}) or {} litellm_opik_metadata.update(requester_opik_metadata) From 647f2f5d86401441928ecbb671be3152ade5d0b3 Mon Sep 17 00:00:00 2001 From: Ariel Date: Tue, 28 Oct 2025 04:51:29 +0200 Subject: [PATCH 29/91] [feat]: graceful degradation for pillar service when using litellm (#15857) * graceful degradation for pillar service when using litellm * remove unnecessary mode * simplify docs * final fixes * lint fixes * fix linting --- .../docs/proxy/guardrails/pillar_security.md | 66 ++++++- .../guardrail_hooks/pillar/__init__.py | 4 + .../guardrail_hooks/pillar/pillar.py | 162 ++++++++++-------- .../guardrails/guardrail_hooks/pillar.py | 8 + .../guardrails/test_pillar_guardrails.py | 123 ++++++++++++- 5 files changed, 289 insertions(+), 74 deletions(-) diff --git a/docs/my-website/docs/proxy/guardrails/pillar_security.md b/docs/my-website/docs/proxy/guardrails/pillar_security.md index a5a416839f6..5ab9f9bf8cb 100644 --- a/docs/my-website/docs/proxy/guardrails/pillar_security.md +++ b/docs/my-website/docs/proxy/guardrails/pillar_security.md @@ -29,7 +29,7 @@ Use Pillar Security for comprehensive LLM security including: Add Pillar Security to your `config.yaml`: -**🌟 Recommended Configuration (Dual Mode):** +**🌟 Recommended Configuration:** ```yaml model_list: - model_name: gpt-4.1-mini @@ -45,6 +45,8 @@ guardrails: api_key: os.environ/PILLAR_API_KEY # Your Pillar API key api_base: os.environ/PILLAR_API_BASE # Pillar API endpoint on_flagged_action: "monitor" # Log threats but allow requests + fallback_on_error: "allow" # Gracefully degrade if Pillar is down (default) + timeout: 5.0 # Timeout for Pillar API calls in seconds (default) persist_session: true # Keep conversations visible in Pillar dashboard async_mode: false # Request synchronous verdicts include_scanners: true # Return scanner category breakdown @@ -207,6 +209,8 @@ You can configure Pillar Security using environment variables: export PILLAR_API_KEY="your_api_key_here" export PILLAR_API_BASE="https://api.pillar.security" export PILLAR_ON_FLAGGED_ACTION="monitor" +export PILLAR_FALLBACK_ON_ERROR="allow" +export PILLAR_TIMEOUT="30.0" ``` ### Session Tracking @@ -245,6 +249,66 @@ Logs the violation but allows the request to proceed: on_flagged_action: "monitor" ``` +### Resilience and Error Handling + +#### Graceful Degradation (`fallback_on_error`) + +Control what happens when the Pillar API is unavailable (network errors, timeouts, service outages): + +```yaml +fallback_on_error: "allow" # Default - recommended for production resilience +``` + +**Available Options:** + +- **`allow` (Default - Recommended)**: Proceed without scanning when Pillar is unavailable + - **No service interruption** if Pillar is down + - **Best for production** where availability is critical + - Security scans are skipped during outages (logged as warnings) + + ```yaml + guardrails: + - guardrail_name: "pillar-resilient" + litellm_params: + guardrail: pillar + fallback_on_error: "allow" # Graceful degradation + ``` + +- **`block`**: Reject all requests when Pillar is unavailable + - **Fail-secure approach** - no request proceeds without scanning + - **Service interruption** during Pillar outages + - Returns 503 Service Unavailable error + + ```yaml + guardrails: + - guardrail_name: "pillar-fail-secure" + litellm_params: + guardrail: pillar + fallback_on_error: "block" # Fail secure + ``` + +#### Timeout Configuration + +Configure how long to wait for Pillar API responses: + +**Example Configurations:** + +```yaml +# Production: Default - Fast with graceful degradation +guardrails: + - guardrail_name: "pillar-production" + litellm_params: + guardrail: pillar + timeout: 5.0 # Default - fast failure detection + fallback_on_error: "allow" # Graceful degradation (required) +``` + +**Environment Variables:** +```bash +export PILLAR_FALLBACK_ON_ERROR="allow" +export PILLAR_TIMEOUT="5.0" +``` + ## Advanced Configuration **Quick takeaways** diff --git a/litellm/proxy/guardrails/guardrail_hooks/pillar/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/pillar/__init__.py index 29ede085ed6..2e4213a34dd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pillar/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pillar/__init__.py @@ -44,6 +44,10 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" include_evidence=_get_config_value( litellm_params, optional_params, "include_evidence" ), + fallback_on_error=_get_config_value( + litellm_params, optional_params, "fallback_on_error" + ), + timeout=_get_config_value(litellm_params, optional_params, "timeout"), ) litellm.logging_callback_manager.add_litellm_callback(_pillar_callback) diff --git a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py index 1646e39b24d..812fc6a5767 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py +++ b/litellm/proxy/guardrails/guardrail_hooks/pillar/pillar.py @@ -60,7 +60,10 @@ class PillarGuardrail(CustomGuardrail): SUPPORTED_ON_FLAGGED_ACTIONS = ["block", "monitor"] DEFAULT_ON_FLAGGED_ACTION = "monitor" + SUPPORTED_FALLBACK_ACTIONS = ["allow", "block"] + DEFAULT_FALLBACK_ACTION = "allow" BASE_API_URL = "https://api.pillar.security" + DEFAULT_TIMEOUT = 5.0 # 5 seconds - fast failure detection with graceful degradation def __init__( self, @@ -72,6 +75,8 @@ class PillarGuardrail(CustomGuardrail): persist_session: Optional[bool] = None, include_scanners: Optional[bool] = None, include_evidence: Optional[bool] = None, + fallback_on_error: Optional[str] = None, + timeout: Optional[float] = None, **kwargs, ) -> None: """ @@ -82,11 +87,11 @@ class PillarGuardrail(CustomGuardrail): api_key: Pillar API key api_base: Pillar API base URL on_flagged_action: Action to take when content is flagged ('block' or 'monitor') + fallback_on_error: Action when API errors occur ('allow' or 'block') + timeout: Timeout for API calls in seconds **kwargs: Additional arguments passed to parent class """ - self.async_handler = get_async_httpx_client( - llm_provider=httpxSpecialProvider.GuardrailCallback - ) + self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) self.api_key = api_key or os.environ.get("PILLAR_API_KEY") if self.api_key is None: @@ -104,14 +109,10 @@ class PillarGuardrail(CustomGuardrail): self.on_flagged_action = action else: if action: - verbose_proxy_logger.warning( - f"Invalid action '{action}', using default" - ) + verbose_proxy_logger.warning(f"Invalid action '{action}', using default") self.on_flagged_action = self.DEFAULT_ON_FLAGGED_ACTION - verbose_proxy_logger.debug( - f"Pillar Guardrail: Initialized with on_flagged_action: {self.on_flagged_action}" - ) + verbose_proxy_logger.debug(f"Pillar Guardrail: Initialized with on_flagged_action: {self.on_flagged_action}") self.async_mode = self._resolve_bool_config( provided_value=async_mode, @@ -138,6 +139,32 @@ class PillarGuardrail(CustomGuardrail): setting_name="include_evidence", ) + # Validate and set fallback_on_error + action = fallback_on_error or os.environ.get("PILLAR_FALLBACK_ON_ERROR") + if action and action in self.SUPPORTED_FALLBACK_ACTIONS: + self.fallback_on_error = action + else: + if action: + verbose_proxy_logger.warning( + f"Invalid fallback action '{action}', using default '{self.DEFAULT_FALLBACK_ACTION}'" + ) + self.fallback_on_error = self.DEFAULT_FALLBACK_ACTION + + verbose_proxy_logger.debug(f"Pillar Guardrail: Initialized with fallback_on_error: {self.fallback_on_error}") + + # Set timeout with graceful fallback on invalid configuration + if timeout is not None: + self.timeout = timeout + else: + try: + self.timeout = float(os.environ.get("PILLAR_TIMEOUT", str(self.DEFAULT_TIMEOUT))) + except (ValueError, TypeError): + verbose_proxy_logger.warning( + f"Pillar Guardrail: Invalid PILLAR_TIMEOUT value '{os.environ.get('PILLAR_TIMEOUT')}', " + f"falling back to default {self.DEFAULT_TIMEOUT}s" + ) + self.timeout = self.DEFAULT_TIMEOUT + # Define supported event hooks supported_event_hooks = [ GuardrailEventHooks.pre_call, @@ -191,18 +218,14 @@ class PillarGuardrail(CustomGuardrail): """ event_type = GuardrailEventHooks.pre_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: - verbose_proxy_logger.debug( - f"Pillar Guardrail: Pre-call scanning disabled for {self.guardrail_name}" - ) + verbose_proxy_logger.debug(f"Pillar Guardrail: Pre-call scanning disabled for {self.guardrail_name}") return data verbose_proxy_logger.debug("Pillar Guardrail: Pre-call hook") result = await self.run_pillar_guardrail(data) # Add guardrail name to response headers - add_guardrail_to_applied_guardrails_header( - request_data=data, guardrail_name=self.guardrail_name - ) + add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) return result @@ -238,18 +261,14 @@ class PillarGuardrail(CustomGuardrail): """ event_type = GuardrailEventHooks.during_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: - verbose_proxy_logger.debug( - f"Pillar Guardrail: During-call scanning disabled for {self.guardrail_name}" - ) + verbose_proxy_logger.debug(f"Pillar Guardrail: During-call scanning disabled for {self.guardrail_name}") return data verbose_proxy_logger.debug("Pillar Guardrail: During-call moderation hook") result = await self.run_pillar_guardrail(data) # Add guardrail name to response headers - add_guardrail_to_applied_guardrails_header( - request_data=data, guardrail_name=self.guardrail_name - ) + add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) return result @@ -276,9 +295,7 @@ class PillarGuardrail(CustomGuardrail): """ event_type = GuardrailEventHooks.post_call if self.should_run_guardrail(data=data, event_type=event_type) is not True: - verbose_proxy_logger.debug( - f"Pillar Guardrail: Post-call scanning disabled for {self.guardrail_name}" - ) + verbose_proxy_logger.debug(f"Pillar Guardrail: Post-call scanning disabled for {self.guardrail_name}") return response verbose_proxy_logger.debug("Pillar Guardrail: Post-call hook") @@ -286,15 +303,11 @@ class PillarGuardrail(CustomGuardrail): # Extract response messages in the format Pillar expects response_dict = response.model_dump() if hasattr(response, "model_dump") else {} # type: ignore[union-attr] response_messages = [ - choice.get("message") - for choice in response_dict.get("choices", []) - if choice.get("message") + choice.get("message") for choice in response_dict.get("choices", []) if choice.get("message") ] if not response_messages: - verbose_proxy_logger.debug( - "Pillar Guardrail: No response content to scan, skipping post-call analysis" - ) + verbose_proxy_logger.debug("Pillar Guardrail: No response content to scan, skipping post-call analysis") return response # Create complete conversation: original messages + response messages @@ -305,9 +318,7 @@ class PillarGuardrail(CustomGuardrail): await self.run_pillar_guardrail(post_call_data) # Add guardrail name to response headers - add_guardrail_to_applied_guardrails_header( - request_data=data, guardrail_name=self.guardrail_name - ) + add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=self.guardrail_name) return response @@ -326,14 +337,11 @@ class PillarGuardrail(CustomGuardrail): Original data if safe or in monitor mode Raises: - PillarGuardrailAPIError: If the Pillar API call fails - HTTPException: If content is flagged and action is 'block' + HTTPException: If content is flagged and action is 'block', or if API fails and fallback_on_error is 'block' """ # Check if messages are present if not data.get("messages"): - verbose_proxy_logger.debug( - "Pillar Guardrail: No messages detected, bypassing security scan" - ) + verbose_proxy_logger.debug("Pillar Guardrail: No messages detected, bypassing security scan") return data try: @@ -350,19 +358,51 @@ class PillarGuardrail(CustomGuardrail): return data except Exception as e: + # If it's already an HTTPException from content being flagged, re-raise it if isinstance(e, HTTPException): raise e - verbose_proxy_logger.error( - f"Pillar Guardrail: API communication failed - {str(e)}" - ) - raise PillarGuardrailAPIError( - f"Pillar Guardrail scan failed - unable to verify request safety: {str(e)}" - ) + + # Handle API communication errors based on fallback_on_error setting + verbose_proxy_logger.error(f"Pillar Guardrail: API communication failed - {str(e)}") + + return self._handle_api_error(e, data) # ========================================================================= # PRIVATE HELPER METHODS (In logical order of usage) # ========================================================================= + def _handle_api_error(self, error: Exception, data: dict) -> dict: + """ + Handle API errors based on fallback_on_error configuration. + + Args: + error: The exception that occurred during API communication + data: Original request data + + Returns: + Original data if fallback_on_error is 'allow' + + Raises: + HTTPException: If fallback_on_error is 'block' + """ + if self.fallback_on_error == "allow": + verbose_proxy_logger.warning( + "Pillar Guardrail: API unavailable, proceeding without scanning (fallback_on_error=allow)" + ) + return data + else: # fallback_on_error == "block" + verbose_proxy_logger.warning( + "Pillar Guardrail: API unavailable, blocking request (fallback_on_error=block)" + ) + raise HTTPException( + status_code=503, + detail={ + "error": "Pillar Security Guardrail Unavailable", + "message": "Security scanning service is temporarily unavailable and fallback is set to block", + "original_error": str(error), + }, + ) + def _prepare_headers(self) -> Dict[str, str]: """Prepare headers for the Pillar API request.""" if not self.api_key: @@ -385,9 +425,7 @@ class PillarGuardrail(CustomGuardrail): return headers - def _set_bool_header( - self, headers: Dict[str, str], header_name: str, value: Optional[bool] - ) -> None: + def _set_bool_header(self, headers: Dict[str, str], header_name: str, value: Optional[bool]) -> None: """Apply a boolean value as a lowercase string HTTP header when provided.""" if value is None: @@ -520,9 +558,7 @@ class PillarGuardrail(CustomGuardrail): ) return payload - async def _call_pillar_api( - self, headers: Dict[str, str], payload: Dict[str, Any] - ) -> Dict[str, Any]: + async def _call_pillar_api(self, headers: Dict[str, str], payload: Dict[str, Any]) -> Dict[str, Any]: """ Call the Pillar API and return the response. @@ -540,21 +576,17 @@ class PillarGuardrail(CustomGuardrail): url=f"{self.api_base}/api/v1/protect", headers=headers, json=payload, - timeout=30.0, + timeout=self.timeout, ) response.raise_for_status() res = response.json() flagged = res.get("flagged") session_id = res.get("session_id") - verbose_proxy_logger.debug( - f"Pillar Guardrail: Analysis complete - flagged={flagged}, session={session_id}" - ) + verbose_proxy_logger.debug(f"Pillar Guardrail: Analysis complete - flagged={flagged}, session={session_id}") return res - def _process_pillar_response( - self, pillar_response: Dict[str, Any], original_data: dict - ) -> None: + def _process_pillar_response(self, pillar_response: Dict[str, Any], original_data: dict) -> None: """ Process the Pillar API response and handle detections based on configuration. @@ -573,9 +605,7 @@ class PillarGuardrail(CustomGuardrail): # Store session_id from Pillar response for potential reuse pillar_session_id = pillar_response.get("session_id") if pillar_session_id: - verbose_proxy_logger.debug( - f"Pillar Guardrail: Received session_id from server: {pillar_session_id}" - ) + verbose_proxy_logger.debug(f"Pillar Guardrail: Received session_id from server: {pillar_session_id}") # Store in request metadata for use in subsequent hooks if "metadata" not in original_data: original_data["metadata"] = {} @@ -587,13 +617,9 @@ class PillarGuardrail(CustomGuardrail): if self.on_flagged_action == "block": self._raise_pillar_detection_exception(pillar_response) elif self.on_flagged_action == "monitor": - verbose_proxy_logger.info( - "Pillar Guardrail: Monitoring mode - allowing flagged content to proceed" - ) + verbose_proxy_logger.info("Pillar Guardrail: Monitoring mode - allowing flagged content to proceed") - def _raise_pillar_detection_exception( - self, pillar_response: Dict[str, Any] - ) -> None: + def _raise_pillar_detection_exception(self, pillar_response: Dict[str, Any]) -> None: """ Raise an HTTPException for Pillar security detections. @@ -613,9 +639,7 @@ class PillarGuardrail(CustomGuardrail): }, } - verbose_proxy_logger.warning( - "Pillar Guardrail: Request blocked - Security threats detected" - ) + verbose_proxy_logger.warning("Pillar Guardrail: Request blocked - Security threats detected") raise HTTPException(status_code=400, detail=error_detail) diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/pillar.py b/litellm/types/proxy/guardrails/guardrail_hooks/pillar.py index 4d0c9ed1cc5..92e76d6693a 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/pillar.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/pillar.py @@ -31,6 +31,14 @@ class PillarGuardrailConfigModelOptionalParams(BaseModel): default=True, description="Include detailed evidence objects in response payloads (sets `plr_evidence` header).", ) + fallback_on_error: Optional[str] = Field( + default=None, + description="Action to take when Pillar API is unavailable or errors: 'allow' (proceed without scanning) or 'block' (reject request with 503 error). If not provided, the `PILLAR_FALLBACK_ON_ERROR` environment variable is checked, defaults to 'allow'.", + ) + timeout: Optional[float] = Field( + default=None, + description="Timeout in seconds for Pillar API calls. If not provided, the `PILLAR_TIMEOUT` environment variable is checked, defaults to 5.0 seconds.", + ) class PillarGuardrailConfigModel( diff --git a/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py b/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py index 67030a8161f..9aecd01dc56 100644 --- a/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/test_pillar_guardrails.py @@ -526,8 +526,12 @@ async def test_empty_messages(pillar_guardrail_instance, user_api_key_dict, dual async def test_api_error_handling( pillar_guardrail_instance, sample_request_data, user_api_key_dict, dual_cache ): - """Test handling of API connection errors.""" - with pytest.raises(PillarGuardrailAPIError) as excinfo: + """Test handling of API connection errors with block fallback.""" + # Note: pillar_guardrail_instance has fallback_on_error defaulting to "allow" + # so this test sets it to "block" to test error handling + pillar_guardrail_instance.fallback_on_error = "block" # Set to block for this test + + with pytest.raises(HTTPException) as excinfo: with patch( "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", side_effect=Exception("Connection error"), @@ -539,8 +543,119 @@ async def test_api_error_handling( call_type="completion", ) - assert "unable to verify request safety" in str(excinfo.value) - assert "Connection error" in str(excinfo.value) + assert excinfo.value.status_code == 503 + assert "Pillar Security Guardrail Unavailable" in str(excinfo.value.detail) + + +@pytest.mark.asyncio +async def test_api_error_fallback_allow(env_setup): + """Test fallback_on_error='allow' allows requests when API is down.""" + guardrail = PillarGuardrail( + guardrail_name="pillar-fallback-allow", + api_key="test-pillar-key", + api_base="https://api.pillar.security", + fallback_on_error="allow", + ) + + sample_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + } + + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + side_effect=Exception("Connection timeout"), + ): + result = await guardrail.async_pre_call_hook( + data=sample_data, + cache=DualCache(), + user_api_key_dict=UserAPIKeyAuth(), + call_type="completion", + ) + + # Should proceed without scanning + assert result == sample_data + + +@pytest.mark.asyncio +async def test_api_error_fallback_block(env_setup): + """Test fallback_on_error='block' blocks requests when API is down.""" + guardrail = PillarGuardrail( + guardrail_name="pillar-fallback-block", + api_key="test-pillar-key", + api_base="https://api.pillar.security", + fallback_on_error="block", + ) + + sample_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}], + } + + with pytest.raises(HTTPException) as excinfo: + with patch( + "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", + side_effect=Exception("Connection timeout"), + ): + await guardrail.async_pre_call_hook( + data=sample_data, + cache=DualCache(), + user_api_key_dict=UserAPIKeyAuth(), + call_type="completion", + ) + + # Should block with 503 Service Unavailable + assert excinfo.value.status_code == 503 + assert "Pillar Security Guardrail Unavailable" in str(excinfo.value.detail) + + +@pytest.mark.asyncio +async def test_custom_timeout_configuration(env_setup): + """Test custom timeout configuration.""" + custom_timeout = 10.0 + guardrail = PillarGuardrail( + guardrail_name="pillar-custom-timeout", + api_key="test-pillar-key", + api_base="https://api.pillar.security", + timeout=custom_timeout, + ) + + assert guardrail.timeout == custom_timeout + + +def test_fallback_on_error_env_variable(monkeypatch): + """Test fallback_on_error can be set via environment variable.""" + monkeypatch.setenv("PILLAR_API_KEY", "test-key") + monkeypatch.setenv("PILLAR_FALLBACK_ON_ERROR", "block") + + guardrail = PillarGuardrail( + guardrail_name="pillar-env-fallback", + ) + + assert guardrail.fallback_on_error == "block" + + +def test_timeout_env_variable(monkeypatch): + """Test timeout can be set via environment variable.""" + monkeypatch.setenv("PILLAR_API_KEY", "test-key") + monkeypatch.setenv("PILLAR_TIMEOUT", "15.0") + + guardrail = PillarGuardrail( + guardrail_name="pillar-env-timeout", + ) + + assert guardrail.timeout == 15.0 + + +def test_invalid_fallback_action_defaults_to_allow(env_setup): + """Test invalid fallback_on_error value defaults to 'allow'.""" + guardrail = PillarGuardrail( + guardrail_name="pillar-invalid-fallback", + api_key="test-pillar-key", + fallback_on_error="invalid_action", + ) + + assert guardrail.fallback_on_error == "allow" @pytest.mark.asyncio From 3a7c498eff64da87fefb72b28acc72a75a1e6b04 Mon Sep 17 00:00:00 2001 From: Deepanshu Lulla Date: Tue, 28 Oct 2025 17:46:50 -0400 Subject: [PATCH 30/91] Add GitlabPromptCache and enable subfolder access (#15712) * Add GitlabPromptCache and enable subfolder access * Add GitlabPromptCache and enable subfolder access * Add GitlabPromptCache and enable subfolder access --------- Co-authored-by: deepanshu --- litellm/integrations/gitlab/__init__.py | 19 +- .../gitlab/gitlab_prompt_manager.py | 180 ++++++++- .../gitlab/test_gitlab_integration.py | 20 +- .../gitlab/test_gitlab_prompt_manager.py | 363 +++++++++++++++++- 4 files changed, 554 insertions(+), 28 deletions(-) diff --git a/litellm/integrations/gitlab/__init__.py b/litellm/integrations/gitlab/__init__.py index cd22afc2ba0..c73a23b6874 100644 --- a/litellm/integrations/gitlab/__init__.py +++ b/litellm/integrations/gitlab/__init__.py @@ -8,7 +8,7 @@ if TYPE_CHECKING: from litellm.types.prompts.init_prompts import SupportedPromptIntegrations from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.types.prompts.init_prompts import PromptSpec, PromptLiteLLMParams -from .gitlab_prompt_manager import GitLabPromptManager +from .gitlab_prompt_manager import GitLabPromptManager, GitLabPromptCache # Global instances global_gitlab_config: Optional[dict] = None @@ -16,13 +16,13 @@ global_gitlab_config: Optional[dict] = None def set_global_gitlab_config(config: dict) -> None: """ - Set the global BitBucket configuration for prompt management. + Set the global gitlab configuration for prompt management. Args: - config: Dictionary containing BitBucket configuration - - workspace: BitBucket workspace name + config: Dictionary containing gitlab configuration + - workspace: gitlab workspace name - repository: Repository name - - access_token: BitBucket access token + - access_token: gitlab access token - branch: Branch to fetch prompts from (default: main) """ import litellm @@ -34,7 +34,7 @@ def prompt_initializer( litellm_params: "PromptLiteLLMParams", prompt_spec: "PromptSpec" ) -> "CustomPromptManagement": """ - Initialize a prompt from a BitBucket repository. + Initialize a prompt from a Gitlab repository. """ gitlab_config = getattr(litellm_params, "gitlab_config", None) prompt_id = getattr(litellm_params, "prompt_id", None) @@ -42,16 +42,16 @@ def prompt_initializer( if not gitlab_config: raise ValueError( - "bitbucket_config is required for BitBucket prompt integration" + "gitlab_config is required for gitlab prompt integration" ) try: - bitbucket_prompt_manager = GitLabPromptManager( + gitlab_prompt_manager = GitLabPromptManager( gitlab_config=gitlab_config, prompt_id=prompt_id, ) - return bitbucket_prompt_manager + return gitlab_prompt_manager except Exception as e: raise e @@ -90,6 +90,7 @@ prompt_initializer_registry = { # Export public API __all__ = [ "GitLabPromptManager", + "GitLabPromptCache", "set_global_gitlab_config", "global_gitlab_config", ] diff --git a/litellm/integrations/gitlab/gitlab_prompt_manager.py b/litellm/integrations/gitlab/gitlab_prompt_manager.py index b782f10ccc5..37013273cb0 100644 --- a/litellm/integrations/gitlab/gitlab_prompt_manager.py +++ b/litellm/integrations/gitlab/gitlab_prompt_manager.py @@ -12,10 +12,24 @@ from litellm.integrations.prompt_management_base import ( ) from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import StandardCallbackDynamicParams - from litellm.integrations.gitlab.gitlab_client import GitLabClient +GITLAB_PREFIX = "gitlab::" + +def encode_prompt_id(raw_id: str) -> str: + """Convert GitLab path IDs like 'invoice/extract' → 'gitlab::invoice::extract'""" + if raw_id.startswith(GITLAB_PREFIX): + return raw_id # already encoded + return f"{GITLAB_PREFIX}{raw_id.replace('/', '::')}" + +def decode_prompt_id(encoded_id: str) -> str: + """Convert 'gitlab::invoice::extract' → 'invoice/extract'""" + if not encoded_id.startswith(GITLAB_PREFIX): + return encoded_id + return encoded_id[len(GITLAB_PREFIX):].replace("::", "/") + + class GitLabPromptTemplate: def __init__( self, @@ -87,6 +101,7 @@ class GitLabTemplateManager: def _id_to_repo_path(self, prompt_id: str) -> str: """Map a prompt_id to a repo path (respects prompts_path and adds .prompt).""" + prompt_id = decode_prompt_id(prompt_id) if self.prompts_path: return f"{self.prompts_path}/{prompt_id}.prompt" return f"{prompt_id}.prompt" @@ -101,26 +116,27 @@ class GitLabTemplateManager: path = path[len(self.prompts_path.strip("/")) + 1 :] if path.endswith(".prompt"): path = path[: -len(".prompt")] - return path + return encode_prompt_id(path) # ---------- loading ---------- def _load_prompt_from_gitlab(self, prompt_id: str, *, ref: Optional[str] = None) -> None: """Load a specific .prompt file from GitLab (scoped under prompts_path if set).""" try: + # prompt_id = decode_prompt_id(prompt_id) file_path = self._id_to_repo_path(prompt_id) prompt_content = self.gitlab_client.get_file_content(file_path, ref=ref) if prompt_content: template = self._parse_prompt_file(prompt_content, prompt_id) self.prompts[prompt_id] = template except Exception as e: - raise Exception(f"Failed to load prompt '{prompt_id}' from GitLab: {e}") + raise Exception(f"Failed to load prompt '{encode_prompt_id(prompt_id)}' from GitLab: {e}") def load_all_prompts(self, *, recursive: bool = True) -> List[str]: """ Eagerly load all .prompt files from prompts_path. Returns loaded IDs. """ - files = self.list_templates(recursive=recursive) # reuse logic + files = self.list_templates(recursive=recursive) loaded: List[str] = [] for pid in files: if pid not in self.prompts: @@ -195,9 +211,6 @@ class GitLabTemplateManager: return self.prompts.get(template_id) def list_templates(self, *, recursive: bool = True) -> List[str]: - """ - List available prompt IDs discovered under prompts_path (no extension, relative to prompts_path). - """ """ List available prompt IDs under prompts_path (no extension). Compatible with both list_files signatures: @@ -248,7 +261,7 @@ class GitLabPromptManager(CustomPromptManagement): "access_token": "glpat_***", "tag": "v1.2.3", # optional; takes precedence "branch": "main", # default fallback - "prompts_path": "prompts/chat" # <--- NEW + "prompts_path": "prompts/chat" } """ @@ -438,9 +451,11 @@ class GitLabPromptManager(CustomPromptManagement): prompt_version: Optional[int] = None, ) -> PromptManagementClient: try: - if prompt_id not in self.prompt_manager.prompts: + decoded_id = decode_prompt_id(prompt_id) + if decoded_id not in self.prompt_manager.prompts: git_ref = getattr(dynamic_callback_params, "extra", {}).get("git_ref") if hasattr(dynamic_callback_params, "extra") else None - self.prompt_manager._load_prompt_from_gitlab(prompt_id, ref=git_ref) + self.prompt_manager._load_prompt_from_gitlab(decoded_id, ref=git_ref) + rendered_prompt, prompt_metadata = self.get_prompt_template( prompt_id, prompt_variables @@ -486,3 +501,148 @@ class GitLabPromptManager(CustomPromptManagement): prompt_label, prompt_version, ) + + +class GitLabPromptCache: + """ + Cache all .prompt files from a GitLab repo into memory. + + - Keys are the *repo file paths* (e.g. "prompts/chat/greet/hi.prompt") + mapped to JSON-like dicts containing content + metadata. + - Also exposes a by-ID view (ID == path relative to prompts_path without ".prompt", + e.g. "greet/hi"). + + Usage: + + cfg = { + "project": "group/subgroup/repo", + "access_token": "glpat_***", + "prompts_path": "prompts/chat", # optional, can be empty for repo root + # "branch": "main", # default is "main" + # "tag": "v1.2.3", # takes precedence over branch + # "base_url": "https://gitlab.com/api/v4" # default + } + + cache = GitLabPromptCache(cfg) + cache.load_all() # fetch + parse all .prompt files + + print(cache.list_files()) # repo file paths + print(cache.list_ids()) # template IDs relative to prompts_path + + prompt_json = cache.get_by_file("prompts/chat/greet/hi.prompt") + prompt_json2 = cache.get_by_id("greet/hi") + + # If GitLab content changes and you want to refresh: + cache.reload() # re-scan and refresh all + """ + + def __init__( + self, + gitlab_config: Dict[str, Any], + *, + ref: Optional[str] = None, + gitlab_client: Optional[GitLabClient] = None, + ) -> None: + # Build a PromptManager (which internally builds TemplateManager + Client) + self.prompt_manager = GitLabPromptManager( + gitlab_config=gitlab_config, + prompt_id=None, + ref=ref, + gitlab_client=gitlab_client, + ) + self.template_manager: GitLabTemplateManager = self.prompt_manager.prompt_manager + + # In-memory stores + self._by_file: Dict[str, Dict[str, Any]] = {} + self._by_id: Dict[str, Dict[str, Any]] = {} + + # ------------------------- + # Public API + # ------------------------- + + def load_all(self, *, recursive: bool = True) -> Dict[str, Dict[str, Any]]: + """ + Scan GitLab for all .prompt files under prompts_path, load and parse each, + and return the mapping of repo file path -> JSON-like dict. + """ + ids = self.template_manager.list_templates(recursive=recursive) # IDs relative to prompts_path + for pid in ids: + # Ensure template is loaded into TemplateManager + if pid not in self.template_manager.prompts: + self.template_manager._load_prompt_from_gitlab(pid) + + tmpl = self.template_manager.get_template(pid) + if tmpl is None: + # If something raced/failed, try once more + self.template_manager._load_prompt_from_gitlab(pid) + tmpl = self.template_manager.get_template(pid) + if tmpl is None: + continue + + file_path = self.template_manager._id_to_repo_path(pid) # "prompts/chat/..../file.prompt" + entry = self._template_to_json(pid, tmpl) + + self._by_file[file_path] = entry + # prefixed_id = pid if pid.startswith("gitlab::") else f"gitlab::{pid}" + encoded_id = encode_prompt_id(pid) + self._by_id[encoded_id] = entry + # self._by_id[pid] = entry + + return self._by_id + + def reload(self, *, recursive: bool = True) -> Dict[str, Dict[str, Any]]: + """Clear the cache and re-load from GitLab.""" + self._by_file.clear() + self._by_id.clear() + return self.load_all(recursive=recursive) + + def list_files(self) -> List[str]: + """Return the repo file paths currently cached.""" + return list(self._by_file.keys()) + + def list_ids(self) -> List[str]: + """Return the template IDs (relative to prompts_path, without extension) currently cached.""" + return list(self._by_id.keys()) + + def get_by_file(self, file_path: str) -> Optional[Dict[str, Any]]: + """Get a cached prompt JSON by repo file path.""" + return self._by_file.get(file_path) + + def get_by_id(self, prompt_id: str) -> Optional[Dict[str, Any]]: + """Get a cached prompt JSON by prompt ID (relative to prompts_path).""" + if prompt_id in self._by_id: + return self._by_id[prompt_id] + + # Try normalized forms + decoded = decode_prompt_id(prompt_id) + encoded = encode_prompt_id(decoded) + + return self._by_id.get(encoded) or self._by_id.get(decoded) + + # ------------------------- + # Internals + # ------------------------- + + def _template_to_json(self, prompt_id: str, tmpl: GitLabPromptTemplate) -> Dict[str, Any]: + """ + Normalize a GitLabPromptTemplate into a JSON-like dict that is easy to serialize. + """ + # Safer copy of metadata (avoid accidental mutation) + md = dict(tmpl.metadata or {}) + + # Pull standard fields (also present in metadata sometimes) + model = tmpl.model + temperature = tmpl.temperature + max_tokens = tmpl.max_tokens + optional_params = dict(tmpl.optional_params or {}) + + return { + "id": prompt_id, # e.g. "greet/hi" + "path": self.template_manager._id_to_repo_path(prompt_id), # e.g. "prompts/chat/greet/hi.prompt" + "content": tmpl.content, # rendered content (without frontmatter) + "metadata": md, # parsed frontmatter + "model": model, + "temperature": temperature, + "max_tokens": max_tokens, + "optional_params": optional_params, + } \ No newline at end of file diff --git a/tests/test_litellm/integrations/gitlab/test_gitlab_integration.py b/tests/test_litellm/integrations/gitlab/test_gitlab_integration.py index 6ad7901459d..ae0de4f4645 100644 --- a/tests/test_litellm/integrations/gitlab/test_gitlab_integration.py +++ b/tests/test_litellm/integrations/gitlab/test_gitlab_integration.py @@ -1,11 +1,14 @@ import os import sys -from unittest.mock import MagicMock, patch + import pytest -sys.path.insert(0, os.path.abspath("../../..")) +sys.path.insert( + 0, os.path.abspath("../../..") +) # Adds the parent directory to the system path +from unittest.mock import MagicMock, patch from litellm.integrations.gitlab.gitlab_prompt_manager import GitLabPromptManager @@ -84,8 +87,9 @@ def test_gitlab_prompt_manager_error_handling_load(mock_client_class): config = {"project": "g/s/r", "access_token": "tkn"} - with pytest.raises(Exception, match="Failed to load prompt 'oops' from GitLab"): - GitLabPromptManager(config, prompt_id="oops").prompt_manager # triggers load + with pytest.raises(Exception, match="Failed to load prompt 'gitlab::oops' from GitLab"): + GitLabPromptManager(config, prompt_id="oops").prompt_manager + def test_gitlab_prompt_manager_config_validation_via_client_ctor(): @@ -257,7 +261,7 @@ def test_gitlab_prompt_manager_list_templates_with_prompts_path(mock_client_clas # list_templates strips folder prefix + extension ids = manager.get_available_prompts() assert "a" in ids - assert "sub/b" in ids + assert "gitlab::sub::b" in ids assert all(not x.endswith(".prompt") for x in ids) assert all("/prompts/chat/" not in x for x in ids) @@ -284,8 +288,8 @@ def test_gitlab_template_manager_load_all_prompts(mock_client_class): pm = GitLabPromptManager(config).prompt_manager loaded = pm.load_all_prompts() - assert set(loaded) == {"a", "sub/b"} - assert "a" in pm.prompts and "sub/b" in pm.prompts + assert set(loaded) == {"gitlab::a", "gitlab::sub::b"} + assert "gitlab::a" in pm.prompts and "gitlab::sub::b" in pm.prompts # ----------------------------- @@ -452,4 +456,4 @@ def test_gitlab_prompt_version_with_prompts_path(mock_client_class): # Path should include prompts_path and end with .prompt mock_client.get_file_content.assert_any_call( "prompts/chat/folder/sub/my_prompt.prompt", ref="commit-sha-999" - ) + ) \ No newline at end of file diff --git a/tests/test_litellm/integrations/gitlab/test_gitlab_prompt_manager.py b/tests/test_litellm/integrations/gitlab/test_gitlab_prompt_manager.py index 5b533fe7653..8475252cfc2 100644 --- a/tests/test_litellm/integrations/gitlab/test_gitlab_prompt_manager.py +++ b/tests/test_litellm/integrations/gitlab/test_gitlab_prompt_manager.py @@ -1,7 +1,6 @@ import os import sys from unittest.mock import MagicMock, patch - import pytest sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to the system path @@ -10,6 +9,10 @@ from litellm.integrations.gitlab.gitlab_client import GitLabClient from litellm.integrations.gitlab.gitlab_prompt_manager import ( GitLabPromptManager, GitLabPromptTemplate, + GitLabTemplateManager, + GitLabPromptCache, + encode_prompt_id, + decode_prompt_id, ) # ----------------------- @@ -475,3 +478,361 @@ def test_gitlab_prompt_manager_version_precedence(mock_client_class): prompt_variables={"q": "hello"}, ) mock_client.get_file_content.assert_any_call("pC.prompt", ref="manager-default") + + + + +# --------------------------------------------------------------------- +# ID Encoding/Decoding helpers +# --------------------------------------------------------------------- + +def test_encode_decode_prompt_id_roundtrip(): + raw = "invoice/extract" + encoded = encode_prompt_id(raw) + assert encoded == "gitlab::invoice::extract" + assert decode_prompt_id(encoded) == raw + +def test_encode_prompt_id_already_encoded(): + encoded = "gitlab::test::path" + assert encode_prompt_id(encoded) == encoded + + +# --------------------------------------------------------------------- +# GitLabTemplateManager behavior +# --------------------------------------------------------------------- + +@pytest.fixture +def mock_gitlab_client(): + client = MagicMock() + client.get_file_content.return_value = """--- +model: bedrock/anthropic.claude-3-sonnet +temperature: 0.3 +max_tokens: 100 +--- +system: You are a helpful bot. +user: Hello {{ name }} +""" + client.list_files.return_value = [ + "prompts/chat/hello.prompt", + "prompts/chat/nested/sub.prompt", + ] + return client + + +@pytest.fixture +def manager(mock_gitlab_client): + cfg = { + "project": "group/repo", + "access_token": "token", + "prompts_path": "prompts/chat", + } + return GitLabTemplateManager(gitlab_config=cfg, gitlab_client=mock_gitlab_client) + + +def test_list_templates_returns_encoded_ids(manager): + ids = manager.list_templates() + assert all(id.startswith("gitlab::") for id in ids) + assert "gitlab::hello" in ids + assert "gitlab::nested::sub" in ids + + +def test_load_prompt_from_gitlab_parses_metadata(manager, mock_gitlab_client): + manager._load_prompt_from_gitlab("gitlab::hello") + assert "gitlab::hello" in manager.prompts + + tmpl = manager.prompts["gitlab::hello"] + assert isinstance(tmpl, GitLabPromptTemplate) + assert tmpl.metadata["model"].startswith("bedrock/") + assert "You are a helpful bot." in tmpl.content + + +def test_render_template_renders_jinja(manager, mock_gitlab_client): + manager._load_prompt_from_gitlab("gitlab::hello") + output = manager.render_template("gitlab::hello", {"name": "Prishu"}) + assert "Hello Prishu" in output + + +def test_get_template_returns_none_if_not_loaded(manager): + assert manager.get_template("gitlab::missing") is None + + +def test_repo_path_conversion(manager): + raw = "gitlab::nested::sub" + repo_path = manager._id_to_repo_path(raw) + assert repo_path.endswith("nested/sub.prompt") + # Ensure decode/encode reversibility + decoded = manager._repo_path_to_id(repo_path) + assert decoded == raw + + +# --------------------------------------------------------------------- +# GitLabPromptManager high-level integration +# --------------------------------------------------------------------- + +@pytest.fixture +def prompt_manager(mock_gitlab_client): + cfg = {"project": "group/repo", "access_token": "tkn", "prompts_path": "prompts/chat"} + return GitLabPromptManager(gitlab_config=cfg, gitlab_client=mock_gitlab_client) + + +def test_get_prompt_template_renders_content(prompt_manager): + encoded_id = "gitlab::hello" + content, meta = prompt_manager.get_prompt_template(encoded_id, {"name": "World"}) + assert "Hello World" in content + assert "model" in meta + + +def test_pre_call_hook_parses_roles(prompt_manager): + prompt_id = "gitlab::hello" + messages, params = prompt_manager.pre_call_hook( + user_id="user123", + messages=[], + prompt_id=prompt_id, + prompt_variables={"name": "Tester"}, + ) + assert isinstance(messages, list) + roles = [m["role"] for m in messages] + assert "system" in roles and "user" in roles + assert "model" in params + + +def test_get_available_prompts_returns_sorted(prompt_manager): + ids = prompt_manager.get_available_prompts() + assert any(id.startswith("gitlab::") for id in ids) + assert ids == sorted(ids) + + +# --------------------------------------------------------------------- +# GitLabPromptCache behavior +# --------------------------------------------------------------------- + +@pytest.fixture +def prompt_cache(mock_gitlab_client): + cfg = {"project": "group/repo", "access_token": "tkn", "prompts_path": "prompts/chat"} + return GitLabPromptCache(cfg, gitlab_client=mock_gitlab_client) + + +def test_cache_load_all_builds_internal_maps(prompt_cache): + result = prompt_cache.load_all() + assert isinstance(result, dict) + # check encoded key presence + assert any(k.startswith("gitlab::") for k in result) + assert prompt_cache.list_files() + assert prompt_cache.list_ids() + + +def test_cache_get_by_id_handles_encoded_and_decoded(prompt_cache): + prompt_cache.load_all() + encoded = "gitlab::hello" + decoded = decode_prompt_id(encoded) + assert prompt_cache.get_by_id(encoded) + assert prompt_cache.get_by_id(decoded) + + +def test_cache_reload_resets_and_reloads(prompt_cache): + prompt_cache.load_all() + before = set(prompt_cache.list_ids()) + prompt_cache.reload() + after = set(prompt_cache.list_ids()) + assert before == after + + +# ----------------------- +# Test fakes / fixtures +# ----------------------- + +class FakeTemplateManager: + """ + Minimal stand-in for GitLabTemplateManager that GitLabPromptCache expects. + """ + def __init__(self, prompts_path="prompts"): + # simulate a configured prompts folder (affects _id_to_repo_path) + self.prompts_path = prompts_path.strip("/") + self.prompts = {} # id -> GitLabPromptTemplate + + # Seeds used by list_templates() + self._discoverable_ids = [] + + # Methods used by GitLabPromptCache.load_all + def list_templates(self, *, recursive: bool = True): + return list(self._discoverable_ids) + + def _load_prompt_from_gitlab(self, pid, ref=None): + # Pretend we fetched and parsed a file; add a basic template if not present + if pid not in self.prompts: + self.prompts[pid] = GitLabPromptTemplate( + template_id=pid, + content=f"User: Hello from {pid}", + metadata={"model": "gpt-4", "temperature": 0.1}, + ) + + def get_template(self, pid): + return self.prompts.get(pid) + + def _id_to_repo_path(self, pid): + base = f"{self.prompts_path}/" if self.prompts_path else "" + return f"{base}{pid}.prompt" + + +class FakePromptManagerWrapper: + """ + Minimal wrapper to mimic GitLabPromptManager(prompt_manager=). + GitLabPromptCache.__init__ expects GitLabPromptManager(...).prompt_manager. + """ + def __init__(self, fake_tm): + self.prompt_manager = fake_tm + + +@pytest.fixture() +def fake_managers(): + """ + Provide a fresh FakeTemplateManager plus a wrapper for each test. + """ + tm = FakeTemplateManager(prompts_path="prompts/chat") + wrapper = FakePromptManagerWrapper(tm) + return tm, wrapper + + +# ----------------------- +# Tests +# ----------------------- + +@patch("litellm.integrations.gitlab.gitlab_prompt_manager.GitLabPromptManager") +def test_cache_load_all_encodes_ids_and_populates_maps(mock_pm_cls, fake_managers): + tm, wrapper = fake_managers + # Simulate two files discovered under prompts_path + tm._discoverable_ids = ["a", "sub/b"] + + # When GitLabPromptCache constructs GitLabPromptManager(...), return our wrapper + mock_pm_cls.return_value = wrapper + + cache = GitLabPromptCache({"project": "g/s/r", "access_token": "tkn"}) + result = cache.load_all() + + # Encoded keys are present + assert set(result.keys()) == {encode_prompt_id("a"), encode_prompt_id("sub/b")} + + # Files map built with full repo paths + expect_a_path = tm._id_to_repo_path("a") + expect_b_path = tm._id_to_repo_path("sub/b") + assert cache.list_files() == [expect_a_path, expect_b_path] + + # IDs list is the encoded IDs + assert set(cache.list_ids()) == {encode_prompt_id("a"), encode_prompt_id("sub/b")} + + # Stored entries have normalized json shape + a_entry = cache.get_by_id("gitlab::a") + assert a_entry["id"] == "a" # id is the raw (decoded) id in the entry body + assert a_entry["path"] == expect_a_path + assert a_entry["metadata"]["model"] == "gpt-4" + + +@patch("litellm.integrations.gitlab.gitlab_prompt_manager.GitLabPromptManager") +def test_cache_get_by_id_accepts_encoded_and_decoded(mock_pm_cls, fake_managers): + tm, wrapper = fake_managers + tm._discoverable_ids = ["x/y"] + mock_pm_cls.return_value = wrapper + + cache = GitLabPromptCache({"project": "g/s/r", "access_token": "tkn"}) + cache.load_all() + + # Encoded lookup + encoded = encode_prompt_id("x/y") + decoded = "x/y" + + by_encoded = cache.get_by_id(encoded) + by_decoded = cache.get_by_id(decoded) + + assert by_encoded is not None + assert by_decoded is not None + assert by_encoded == by_decoded # normalization works + # sanity on shape + assert by_encoded["id"] == "x/y" + assert by_encoded["path"].endswith("prompts/chat/x/y.prompt") + + +@patch("litellm.integrations.gitlab.gitlab_prompt_manager.GitLabPromptManager") +def test_cache_reload_clears_then_reloads(mock_pm_cls, fake_managers): + tm, wrapper = fake_managers + tm._discoverable_ids = ["p1"] + mock_pm_cls.return_value = wrapper + + cache = GitLabPromptCache({"project": "g/s/r", "access_token": "tkn"}) + first = cache.load_all() + assert encode_prompt_id("p1") in first + + # Change discovered ids and ensure reload reflects the change + tm._discoverable_ids = ["p2"] + reloaded = cache.reload() + + assert encode_prompt_id("p1") not in reloaded + assert encode_prompt_id("p2") in reloaded + # internal maps should reflect only new state + assert cache.list_ids() == [encode_prompt_id("p2")] + + +@patch("litellm.integrations.gitlab.gitlab_prompt_manager.GitLabPromptManager") +def test_cache_skips_when_template_missing_even_after_reload_attempt(mock_pm_cls, fake_managers): + """ + If get_template(pid) returns None even after a retry load, the entry is skipped. + """ + class MissingTemplateManager(FakeTemplateManager): + def get_template(self, pid): + # Always return None to trigger the continue path + return None + + def _load_prompt_from_gitlab(self, pid, ref=None): + # Pretend to load, but still don't populate prompts so get_template stays None + pass + + tm = MissingTemplateManager(prompts_path="prompts") + wrapper = FakePromptManagerWrapper(tm) + mock_pm_cls.return_value = wrapper + + cache = GitLabPromptCache({"project": "g/s/r", "access_token": "tkn"}) + tm._discoverable_ids = ["will/vanish"] + out = cache.load_all() + + assert out == {} + assert cache.list_files() == [] + assert cache.list_ids() == [] + + +@patch("litellm.integrations.gitlab.gitlab_prompt_manager.GitLabPromptManager") +def test_cache_get_by_file_returns_exact_entry(mock_pm_cls, fake_managers): + tm, wrapper = fake_managers + tm._discoverable_ids = ["alpha", "nested/beta"] + mock_pm_cls.return_value = wrapper + + cache = GitLabPromptCache({"project": "g/s/r", "access_token": "tkn"}) + cache.load_all() + + alpha_path = tm._id_to_repo_path("alpha") + beta_path = tm._id_to_repo_path("nested/beta") + + alpha = cache.get_by_file(alpha_path) + beta = cache.get_by_file(beta_path) + + assert alpha and alpha["id"] == "alpha" + assert beta and beta["id"] == "nested/beta" + + +@patch("litellm.integrations.gitlab.gitlab_prompt_manager.GitLabPromptManager") +def test_encode_decode_helpers_roundtrip_in_cache_context(mock_pm_cls, fake_managers): + tm, wrapper = fake_managers + tm._discoverable_ids = ["dir1/dir2/item"] + mock_pm_cls.return_value = wrapper + + cache = GitLabPromptCache({"project": "g/s/r", "access_token": "tkn"}) + cache.load_all() + + encoded = encode_prompt_id("dir1/dir2/item") + assert encoded in cache.list_ids() + + # decode → encode → lookup should still work + decoded = decode_prompt_id(encoded) + assert decoded == "dir1/dir2/item" + + got = cache.get_by_id(decoded) + assert got is not None + assert got["id"] == "dir1/dir2/item" \ No newline at end of file From 59189c0579c21c1aae47719c355c8f312b47d837 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 29 Oct 2025 03:18:04 +0530 Subject: [PATCH 31/91] fix errors in videos documentation (#15996) --- .../my-website/docs/providers/azure/videos.md | 12 +- .../docs/providers/openai/videos.md | 10 +- docs/my-website/docs/videos.md | 266 ++++++++++++++---- docs/my-website/sidebars.js | 3 + 4 files changed, 233 insertions(+), 58 deletions(-) diff --git a/docs/my-website/docs/providers/azure/videos.md b/docs/my-website/docs/providers/azure/videos.md index 22e714a8ecf..d088c63f710 100644 --- a/docs/my-website/docs/providers/azure/videos.md +++ b/docs/my-website/docs/providers/azure/videos.md @@ -31,7 +31,7 @@ os.environ["AZURE_OPENAI_API_VERSION"] = "2024-02-15-preview" ### Basic Usage ```python -from litellm import video_generation, video_status, video_retrieval +from litellm import video_generation, video_status, video_content import os import time @@ -68,7 +68,7 @@ while True: time.sleep(10) # Wait 10 seconds before checking again # Download video content when ready -video_bytes = video_retrieval( +video_bytes = video_content( video_id=response.id, model="azure/sora-2" ) @@ -146,10 +146,10 @@ client = openai.OpenAI( ) # request sent to model set on litellm proxy, `litellm --model` -response = client.videos.generations.create( +response = client.videos.create( model="azure-sora-2", prompt="A cat playing with a ball of yarn in a sunny garden", - seconds="8", + seconds=8, size="720x1280" ) @@ -210,7 +210,7 @@ general_settings: ```python # Download video content -video_bytes = video_retrieval( +video_bytes = video_content( video_id="video_1234567890", model="azure/sora-2" ) @@ -242,7 +242,7 @@ def generate_and_download_video(prompt): time.sleep(30) # Step 3: Download video - video_bytes = litellm.video_retrieval( + video_bytes = litellm.video_content( video_id=video_id, model="azure/sora-2" ) diff --git a/docs/my-website/docs/providers/openai/videos.md b/docs/my-website/docs/providers/openai/videos.md index 4dc18da77bd..473279e60ef 100644 --- a/docs/my-website/docs/providers/openai/videos.md +++ b/docs/my-website/docs/providers/openai/videos.md @@ -17,7 +17,7 @@ os.environ["OPENAI_API_KEY"] = "your-api-key" ### Basic Usage ```python -from litellm import video_generation, video_retrieval +from litellm import video_generation, video_content import os os.environ["OPENAI_API_KEY"] = "your-api-key" @@ -34,7 +34,7 @@ print(f"Video ID: {response.id}") print(f"Status: {response.status}") # Download video content when ready -video_bytes = video_retrieval( +video_bytes = video_content( video_id=response.id, model="sora-2" ) @@ -63,7 +63,7 @@ with open("generated_video.mp4", "wb") as f: ```python # Download video content -video_bytes = video_retrieval( +video_bytes = video_content( video_id="video_1234567890", model="sora-2" ) @@ -95,7 +95,7 @@ def generate_and_download_video(prompt): time.sleep(30) # Step 3: Download video - video_bytes = litellm.video_retrieval( + video_bytes = litellm.video_content( video_id=video_id, model="sora-2" ) @@ -118,7 +118,7 @@ video_file = generate_and_download_video( # Video editing with reference image response = litellm.video_generation( prompt="Make the cat jump higher", - input_reference="path/to/image.jpg", # Reference image + input_reference=open("path/to/image.jpg", "rb"), # Reference image model="sora-2", seconds="8" ) diff --git a/docs/my-website/docs/videos.md b/docs/my-website/docs/videos.md index dd96398f8d6..344a0f852df 100644 --- a/docs/my-website/docs/videos.md +++ b/docs/my-website/docs/videos.md @@ -21,7 +21,7 @@ LiteLLM follows the [OpenAI Video Generation API specification](https://platform ### Quick Start ```python -from litellm import video_generation, video_status, video_retrieval +from litellm import video_generation, video_status, video_content import os import time @@ -56,7 +56,7 @@ while True: time.sleep(10) # Wait 10 seconds before checking again # Download video content when ready -video_bytes = video_retrieval( +video_bytes = video_content( video_id=response.id, model="openai/sora-2" ) @@ -69,7 +69,7 @@ with open("generated_video.mp4", "wb") as f: ### Async Usage ```python -from litellm import avideo_generation, avideo_status, avideo_retrieval +from litellm import avideo_generation, avideo_status, avideo_content import os, asyncio os.environ["OPENAI_API_KEY"] = "sk-.." @@ -103,7 +103,7 @@ async def test_async_video(): await asyncio.sleep(10) # Wait 10 seconds before checking again # Download video content when ready - video_bytes = await avideo_retrieval( + video_bytes = await avideo_content( video_id=response.id, model="openai/sora-2" ) @@ -241,57 +241,45 @@ litellm --config /path/to/config.yaml Test video generation request ```bash -curl http://0.0.0.0:4000/videos/generations \ - -H "Authorization: Bearer sk-1234" \ - -H "Content-Type: application/json" \ - -d '{ +curl --location 'http://localhost:4000/v1/videos' \ +--header 'Content-Type: application/json' \ +--header 'x-litellm-api-key: sk-1234' \ +--data '{ "model": "sora-2", - "prompt": "A cat playing with a ball of yarn in a sunny garden", - "seconds": "8", - "size": "720x1280" - }' + "prompt": "A beautiful sunset over the ocean" +}' ``` Test video status request ```bash -curl http://0.0.0.0:4000/videos/status \ - -H "Authorization: Bearer sk-1234" \ - -H "Content-Type: application/json" \ - -d '{ - "video_id": "video_1234567890", - "model": "sora-2" - }' +curl --location 'http://localhost:4000/v1/videos/video_id' \ +--header 'Accept: application/json' \ +--header 'x-litellm-api-key: sk-1234' + ``` Test video retrieval request ```bash -curl http://0.0.0.0:4000/videos/retrieval \ - -H "Authorization: Bearer sk-1234" \ - -H "Content-Type: application/json" \ - -d '{ - "video_id": "video_1234567890", - "model": "sora-2" - }' +curl --location 'http://localhost:4000/v1/videos/video_id/content' \ +--header 'Accept: application/json' \ +--header 'x-litellm-api-key: sk-1234' + ``` Test video remix request ```bash -curl http://0.0.0.0:4000/videos/remix \ - -H "Authorization: Bearer sk-1234" \ - -H "Content-Type: multipart/form-data" \ - -F 'model=sora-2' \ - -F 'prompt=Make the cat jump higher' \ - -F 'input_reference=@path/to/image.jpg' \ - -F 'seconds=8' +curl --location --request POST 'http://localhost:4000/v1/videos/string/remix' \ +--header 'Accept: application/json' \ +--header 'x-litellm-api-key: sk-1234' ``` Test Azure video generation request ```bash -curl http://0.0.0.0:4000/videos/generations \ +curl http://localhost:4000/v1/videos \ -H "Authorization: Bearer sk-1234" \ -H "Content-Type: application/json" \ -d '{ @@ -302,6 +290,188 @@ curl http://0.0.0.0:4000/videos/generations \ }' ``` +## **Using OpenAI Client with LiteLLM Proxy** + +You can use the standard OpenAI Python client to interact with LiteLLM's video endpoints. This provides a familiar interface while leveraging LiteLLM's provider abstraction and proxy features. + +### Setup + +First, configure your OpenAI client to point to your LiteLLM proxy: + +```python +from openai import OpenAI + +# Point the OpenAI client to your LiteLLM proxy +client = OpenAI( + api_key="sk-1234", # Your LiteLLM proxy API key + base_url="http://localhost:4000/v1" # Your LiteLLM proxy URL +) +``` + +### Video Generation + +Generate a new video using the OpenAI client interface: + +```python +# Basic video generation +response = client.videos.create( + model="sora-2", + prompt="A cat playing with a ball of yarn in a sunny garden", + seconds=8, + size="720x1280" +) + +print(f"Video ID: {response.id}") +print(f"Status: {response.status}") +``` + +### Video Generation with Reference Image + +Create a video using a reference image: + +```python +# Video generation with reference image +response = client.videos.create( + model="sora-2", + prompt="Add clouds to the video", + seconds=4, + input_reference=open("/path/to/your/image.jpg", "rb") +) + +print(f"Video ID: {response.id}") +print(f"Status: {response.status}") +``` + +### Video Status Checking + +Check the status of a video generation: + +```python +# Check video status +status_response = client.videos.retrieve( + video_id="video_6900378779308191a7359266e59b53fc01cd6bbd27a70763" +) + +print(f"Status: {status_response.status}") +print(f"Progress: {status_response.progress}%") + +# Poll until completion +import time + +while status_response.status not in ["completed", "failed"]: + time.sleep(10) # Wait 10 seconds + status_response = client.videos.retrieve( + video_id="video_6900378779308191a7359266e59b53fc01cd6bbd27a70763" + ) + print(f"Current status: {status_response.status}") +``` + +### List Videos + +Get a list of your videos: + +```python +# List all videos +videos = client.videos.list() + +for video in videos.data: + print(f"Video ID: {video.id}, Status: {video.status}") +``` + +### Download Video Content + +Download the completed video: + +```python +# Download video content +response = client.videos.download_content( + video_id="video_68fa2938848c8190bb718f977503aba6092ab18d68938fed" +) + +# Save the video to file +with open("generated_video.mp4", "wb") as f: + f.write(response.content) + +print("Video downloaded successfully!") +``` + +### Video Remix (Editing) + +Edit an existing video with new instructions: + +```python +# Remix/edit an existing video +response = client.videos.remix( + video_id="video_68fa2574bdd88190873a8af06a370ff407094ddbc4bbb91b", + prompt="Slow the cloud movement", + seconds=8 +) + +print(f"Remix Video ID: {response.id}") +print(f"Status: {response.status}") +``` + +### Complete Workflow Example + +Here's a complete example showing the full video generation workflow: + +```python +from openai import OpenAI +import time + +# Initialize client +client = OpenAI( + api_key="sk-1234", + base_url="http://localhost:4000/v1" +) + +# 1. Generate video +print("Generating video...") +response = client.videos.create( + model="sora-2", + prompt="A serene lake with mountains in the background", + seconds=8, + size="1280x720" +) + +video_id = response.id +print(f"Video generation started. ID: {video_id}") + +# 2. Poll for completion +print("Waiting for video to complete...") +while True: + status = client.videos.retrieve(video_id=video_id) + print(f"Status: {status.status}") + + if status.status == "completed": + print("Video generation completed!") + break + elif status.status == "failed": + print("Video generation failed!") + break + + time.sleep(10) + +# 3. Download video +if status.status == "completed": + print("Downloading video...") + video_content = client.videos.download_content(video_id=video_id) + + with open(f"video_{video_id}.mp4", "wb") as f: + f.write(video_content.content) + + print("Video saved successfully!") + +# 4. Optional: Remix the video +print("Creating a remix...") +remix_response = client.videos.remix( + video_id=video_id, + prompt="Add gentle ripples to the lake surface" +) + +print(f"Remix started. ID: {remix_response.id}") +``` + ## **Request/Response Format** :::info @@ -381,19 +551,21 @@ The response follows OpenAI's video generation format with the following structu ```json { - "id": "video_1234567890", - "object": "video", - "status": "queued", - "created_at": 1712697600, - "model": "sora-2", - "size": "720x1280", - "seconds": "8", - "usage": { - "prompt_tokens": 0, - "completion_tokens": 0, - "total_tokens": 0, - "duration_seconds": 8.0 - } + "id": "video_6900378779308191a7359266e59b53fc01cd6bbd27a70763", + "object": "video", + "status": "queued", + "created_at": 1761621895, + "completed_at": null, + "expires_at": null, + "error": null, + "progress": 0, + "remixed_from_video_id": null, + "seconds": "4", + "size": "720x1280", + "model": "sora-2", + "usage": { + "duration_seconds": 4.0 + } } ``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 82ef79c7870..f83f454aa91 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -432,6 +432,7 @@ const sidebars = { "providers/openai", "providers/openai/responses_api", "providers/openai/text_to_speech", + "providers/openai/videos", ] }, "providers/text_completion_openai", @@ -444,6 +445,7 @@ const sidebars = { "providers/azure/azure_responses", "providers/azure/azure_embedding", "providers/azure/azure_speech", + "providers/azure/videos", ] }, { @@ -714,6 +716,7 @@ const sidebars = { items: [ "adding_provider/directory_structure", "adding_provider/new_rerank_provider", + ] }, "extras/contributing", "contributing", From 12de66dad61764ae60e52c15376b1834e3175c98 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 28 Oct 2025 15:27:10 -0700 Subject: [PATCH 32/91] Config Models should not be editable (#16020) --- .../src/components/model_info_view.test.tsx | 179 ++++++++++++++++++ .../src/components/model_info_view.tsx | 21 +- 2 files changed, 195 insertions(+), 5 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/model_info_view.test.tsx diff --git a/ui/litellm-dashboard/src/components/model_info_view.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.test.tsx new file mode 100644 index 00000000000..3b36e7324cc --- /dev/null +++ b/ui/litellm-dashboard/src/components/model_info_view.test.tsx @@ -0,0 +1,179 @@ +import { render, waitFor } from "@testing-library/react"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import ModelInfoView from "./model_info_view"; + +vi.mock("../../utils/dataUtils", () => ({ + copyToClipboard: vi.fn(), +})); + +vi.mock("./networking", () => ({ + modelInfoV1Call: vi.fn().mockResolvedValue({ + data: [ + { + model_name: "aws/anthropic/bedrock-claude-3-5-sonnet-v1", + litellm_params: { + aws_region_name: "us-east-1", + custom_llm_provider: "bedrock", + use_in_pass_through: false, + use_litellm_proxy: false, + merge_reasoning_content_in_choices: false, + model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + }, + model_info: { + id: "70b94bbd2af4a75215f7e3e465b5b199529dc15deb5d395d0668a4aabc496c84", + db_model: false, + access_via_team_ids: [ + "4fe3cfea-c907-412a-a645-60915b618d11", + "9a4b2d15-4198-47e4-971b-7329b77f40e4", + "14d55eef-b8d4-4cb8-b080-d973269dae54", + "693ce1d2-9fae-4605-a5c9-1c9829415e1a", + "fe29d910-4968-45bc-9fe0-6716e89c6270", + ], + direct_access: true, + key: "anthropic.claude-3-5-sonnet-20240620-v1:0", + max_tokens: 4096, + max_input_tokens: 200000, + max_output_tokens: 4096, + input_cost_per_token: 0.000003, + input_cost_per_token_flex: null, + input_cost_per_token_priority: null, + cache_creation_input_token_cost: null, + cache_read_input_token_cost: null, + cache_read_input_token_cost_flex: null, + cache_read_input_token_cost_priority: null, + cache_creation_input_token_cost_above_1hr: null, + input_cost_per_character: null, + input_cost_per_token_above_128k_tokens: null, + input_cost_per_token_above_200k_tokens: null, + input_cost_per_query: null, + input_cost_per_second: null, + input_cost_per_audio_token: null, + input_cost_per_token_batches: null, + output_cost_per_token_batches: null, + output_cost_per_token: 0.000015, + output_cost_per_token_flex: null, + output_cost_per_token_priority: null, + output_cost_per_audio_token: null, + output_cost_per_character: null, + output_cost_per_reasoning_token: null, + output_cost_per_token_above_128k_tokens: null, + output_cost_per_character_above_128k_tokens: null, + output_cost_per_token_above_200k_tokens: null, + output_cost_per_second: null, + output_cost_per_video_per_second: null, + output_cost_per_image: null, + output_vector_size: null, + citation_cost_per_token: null, + tiered_pricing: null, + litellm_provider: "bedrock", + mode: "chat", + supports_system_messages: null, + supports_response_schema: true, + supports_vision: true, + supports_function_calling: true, + supports_tool_choice: true, + supports_assistant_prefill: null, + supports_prompt_caching: null, + supports_audio_input: null, + supports_audio_output: null, + supports_pdf_input: true, + supports_embedding_image_input: null, + supports_native_streaming: null, + supports_web_search: null, + supports_url_context: null, + supports_reasoning: null, + supports_computer_use: null, + search_context_cost_per_query: null, + tpm: null, + rpm: null, + ocr_cost_per_page: null, + annotation_cost_per_page: null, + supported_openai_params: [ + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "stop", + "temperature", + "top_p", + "extra_headers", + "response_format", + "requestMetadata", + "tools", + "tool_choice", + ], + }, + }, + ], + }), + credentialGetCall: vi.fn().mockResolvedValue({}), +})); + +describe("ModelInfoView", () => { + const modelData = { + model_info: { + id: "123", + created_by: "123", + db_model: true, + }, + litellm_params: { + api_base: "https://api.openai.com/v1", + custom_llm_provider: "openai", + }, + litellm_model_name: "gpt-4", + model_name: "GPT-4", + litellm_provider: "openai", + mode: "chat", + supported_openai_params: ["temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty"], + }; + + it("should render the model info view", async () => { + const { getByText } = render( + {}} + modelData={modelData} + accessToken="123" + userID="123" + userRole="Admin" + editModel={false} + setEditModalVisible={() => {}} + setSelectedModel={() => {}} + onModelUpdate={() => {}} + modelAccessGroups={[]} + />, + ); + await waitFor(() => { + expect(getByText("Model Settings")).toBeInTheDocument(); + }); + }); + + it("should not render an edit model button if the model is not a DB model", async () => { + const nonDbModelData = { + ...modelData, + model_info: { + ...modelData.model_info, + db_model: false, + }, + }; + + const { queryByText } = render( + {}} + modelData={nonDbModelData} + accessToken="123" + userID="123" + userRole="Admin" + editModel={false} + setEditModalVisible={() => {}} + setSelectedModel={() => {}} + onModelUpdate={() => {}} + modelAccessGroups={[]} + />, + ); + await waitFor(() => { + expect(queryByText("Edit Model")).not.toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 89a76eb35a7..23c8f0a9644 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -73,7 +73,8 @@ export default function ModelInfoView({ const [copiedStates, setCopiedStates] = useState>({}); const [isAutoRouterModalOpen, setIsAutoRouterModalOpen] = useState(false); const [guardrailsList, setGuardrailsList] = useState([]); - const canEditModel = userRole === "Admin" || modelData?.model_info?.created_by === userID; + const canEditModel = + (userRole === "Admin" || modelData?.model_info?.created_by === userID) && modelData?.model_info?.db_model; const isAdmin = userRole === "Admin"; const isAutoRouter = modelData?.litellm_params?.auto_router_config != null; @@ -429,10 +430,20 @@ export default function ModelInfoView({ Edit Auto Router )} - {canEditModel && !isEditing && ( - setIsEditing(true)} className="flex items-center"> - Edit Model - + {canEditModel ? ( + !isEditing && ( + setIsEditing(true)} + className="flex items-center" + > + Edit Model + + ) + ) : ( + + + )} From 5c375b23ae2421ffe57499aeba33b4f5b27c3e77 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 28 Oct 2025 16:40:49 -0700 Subject: [PATCH 33/91] [Fix] Guardrails - Ensure Key Guardrails are applied (#16025) * _add_guardrails_from_key_or_team_metadata * test_team_guardrails_append_to_key_guardrails * fix move_guardrails_to_metadata * fix _add_guardrails_from_key_or_team_metadata --- ...odel_prices_and_context_window_backup.json | 30 +++++- litellm/proxy/litellm_pre_call_utils.py | 53 +++++++---- .../proxy/test_litellm_pre_call_utils.py | 94 +++++++++++++++++++ 3 files changed, 156 insertions(+), 21 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8297cc42ff9..0623bc8a904 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1106,6 +1106,7 @@ "supports_vision": true }, "azure/eu/gpt-4o-2024-08-06": { + "deprecation_date": "2026-02-27", "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, "litellm_provider": "azure", @@ -1122,6 +1123,7 @@ "supports_vision": true }, "azure/eu/gpt-4o-2024-11-20": { + "deprecation_date": "2026-03-01", "cache_creation_input_token_cost": 1.38e-06, "input_cost_per_token": 2.75e-06, "litellm_provider": "azure", @@ -1280,7 +1282,7 @@ }, "azure/global-standard/gpt-4o-2024-08-06": { "cache_read_input_token_cost": 1.25e-06, - "deprecation_date": "2025-08-20", + "deprecation_date": "2026-02-27", "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -1297,7 +1299,7 @@ }, "azure/global-standard/gpt-4o-2024-11-20": { "cache_read_input_token_cost": 1.25e-06, - "deprecation_date": "2025-12-20", + "deprecation_date": "2026-03-01", "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", "max_input_tokens": 128000, @@ -1326,6 +1328,7 @@ "supports_vision": true }, "azure/global/gpt-4o-2024-08-06": { + "deprecation_date": "2026-02-27", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -1342,6 +1345,7 @@ "supports_vision": true }, "azure/global/gpt-4o-2024-11-20": { + "deprecation_date": "2026-03-01", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -1625,6 +1629,7 @@ "supports_web_search": false }, "azure/gpt-4.1-2025-04-14": { + "deprecation_date": "2026-11-04", "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_batches": 1e-06, @@ -1691,6 +1696,7 @@ "supports_web_search": false }, "azure/gpt-4.1-mini-2025-04-14": { + "deprecation_date": "2026-11-04", "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 4e-07, "input_cost_per_token_batches": 2e-07, @@ -1756,6 +1762,7 @@ "supports_vision": true }, "azure/gpt-4.1-nano-2025-04-14": { + "deprecation_date": "2026-11-04", "cache_read_input_token_cost": 2.5e-08, "input_cost_per_token": 1e-07, "input_cost_per_token_batches": 5e-08, @@ -1837,6 +1844,7 @@ "supports_vision": true }, "azure/gpt-4o-2024-08-06": { + "deprecation_date": "2026-02-27", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.5e-06, "litellm_provider": "azure", @@ -1853,6 +1861,7 @@ "supports_vision": true }, "azure/gpt-4o-2024-11-20": { + "deprecation_date": "2026-03-01", "cache_read_input_token_cost": 1.25e-06, "input_cost_per_token": 2.75e-06, "litellm_provider": "azure", @@ -2604,6 +2613,7 @@ "supports_vision": true }, "azure/o3-2025-04-16": { + "deprecation_date": "2026-04-16", "cache_read_input_token_cost": 2.5e-06, "input_cost_per_token": 1e-05, "litellm_provider": "azure", @@ -2832,6 +2842,7 @@ "output_cost_per_token": 0.0 }, "azure/text-embedding-3-small": { + "deprecation_date": "2026-04-30", "input_cost_per_token": 2e-08, "litellm_provider": "azure", "max_input_tokens": 8191, @@ -2870,6 +2881,7 @@ "mode": "audio_speech" }, "azure/us/gpt-4o-2024-08-06": { + "deprecation_date": "2026-02-27", "cache_read_input_token_cost": 1.375e-06, "input_cost_per_token": 2.75e-06, "litellm_provider": "azure", @@ -2886,6 +2898,7 @@ "supports_vision": true }, "azure/us/gpt-4o-2024-11-20": { + "deprecation_date": "2026-03-01", "cache_creation_input_token_cost": 1.38e-06, "input_cost_per_token": 2.75e-06, "litellm_provider": "azure", @@ -4911,7 +4924,7 @@ "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, - "deprecation_date": "2026-02-01", + "deprecation_date": "2026-02-19", "input_cost_per_token": 3e-06, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -4968,7 +4981,6 @@ "cache_creation_input_token_cost": 3e-07, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2025-03-01", "input_cost_per_token": 2.5e-07, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -4988,7 +5000,7 @@ "cache_creation_input_token_cost": 1.875e-05, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 1.5e-06, - "deprecation_date": "2025-03-01", + "deprecation_date": "2026-05-01", "input_cost_per_token": 1.5e-05, "litellm_provider": "anthropic", "max_input_tokens": 200000, @@ -5172,6 +5184,7 @@ "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, + "deprecation_date": "2026-08-05", "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 32000, @@ -5199,6 +5212,7 @@ "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, + "deprecation_date": "2026-05-14", "litellm_provider": "anthropic", "max_input_tokens": 200000, "max_output_tokens": 32000, @@ -5222,6 +5236,7 @@ "tool_use_system_prompt_tokens": 159 }, "claude-sonnet-4-20250514": { + "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -7993,6 +8008,7 @@ "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, + "deprecation_date": "2026-10-15", "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, @@ -12397,6 +12413,7 @@ "supports_tool_choice": true }, "gpt-3.5-turbo-1106": { + "deprecation_date": "2026-09-28", "input_cost_per_token": 1e-06, "litellm_provider": "openai", "max_input_tokens": 16385, @@ -12466,6 +12483,7 @@ "supports_tool_choice": true }, "gpt-4-0125-preview": { + "deprecation_date": "2026-03-26", "input_cost_per_token": 1e-05, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -12506,6 +12524,7 @@ "supports_tool_choice": true }, "gpt-4-1106-preview": { + "deprecation_date": "2026-03-26", "input_cost_per_token": 1e-05, "litellm_provider": "openai", "max_input_tokens": 128000, @@ -16563,6 +16582,7 @@ "supports_vision": true }, "o1-mini-2024-09-12": { + "deprecation_date": "2025-10-27", "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 3e-06, "litellm_provider": "openai", diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 1e53655dd0b..7dd5ba2b9ad 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1,7 +1,7 @@ import asyncio import copy import time -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Union from fastapi import Request from starlette.datastructures import Headers @@ -1191,6 +1191,8 @@ def _add_guardrails_from_key_or_team_metadata( ) -> None: """ Helper add guardrails from key or team metadata to request data + + Key guardrails are set first, then team guardrails are appended (without duplicates). Args: key_metadata: The key metadata dictionary to check for guardrails @@ -1201,14 +1203,24 @@ def _add_guardrails_from_key_or_team_metadata( """ from litellm.proxy.utils import _premium_user_check - for _management_object_metadata in [key_metadata, team_metadata]: - if _management_object_metadata and "guardrails" in _management_object_metadata: - if len(_management_object_metadata["guardrails"]) > 0: - _premium_user_check() - - data[metadata_variable_name]["guardrails"] = _management_object_metadata[ - "guardrails" - ] + # Initialize guardrails set (avoiding duplicates) + combined_guardrails = set() + + # Add key-level guardrails first + if key_metadata and "guardrails" in key_metadata: + if isinstance(key_metadata["guardrails"], list) and len(key_metadata["guardrails"]) > 0: + _premium_user_check() + combined_guardrails.update(key_metadata["guardrails"]) + + # Add team-level guardrails (set automatically handles duplicates) + if team_metadata and "guardrails" in team_metadata: + if isinstance(team_metadata["guardrails"], list) and len(team_metadata["guardrails"]) > 0: + _premium_user_check() + combined_guardrails.update(team_metadata["guardrails"]) + + # Set combined guardrails in metadata as list + if combined_guardrails: + data[metadata_variable_name]["guardrails"] = list(combined_guardrails) def move_guardrails_to_metadata( @@ -1230,15 +1242,24 @@ def move_guardrails_to_metadata( metadata_variable_name=_metadata_variable_name, ) - # Check request-level guardrails + ######################################################################################### + # User's might send "guardrails" in the request body, we need to add them to the request metadata. + # Since downstream logic requires "guardrails" to be in the request metadata + ######################################################################################### if "guardrails" in data: - data[_metadata_variable_name]["guardrails"] = data["guardrails"] - del data["guardrails"] - + request_body_guardrails = data.pop("guardrails") + if "guardrails" in data[_metadata_variable_name] and isinstance(data[_metadata_variable_name]["guardrails"], list): + data[_metadata_variable_name]["guardrails"].extend(request_body_guardrails) + else: + data[_metadata_variable_name]["guardrails"] = request_body_guardrails + + ######################################################################################### if "guardrail_config" in data: - data[_metadata_variable_name]["guardrail_config"] = data["guardrail_config"] - del data["guardrail_config"] - + request_body_guardrail_config = data.pop("guardrail_config") + if "guardrail_config" in data[_metadata_variable_name] and isinstance(data[_metadata_variable_name]["guardrail_config"], dict): + data[_metadata_variable_name]["guardrail_config"].update(request_body_guardrail_config) + else: + data[_metadata_variable_name]["guardrail_config"] = request_body_guardrail_config def add_provider_specific_headers_to_request( data: dict, diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index bcc0e5ac2d6..6379ea704ce 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -1140,3 +1140,97 @@ def test_get_sanitized_user_information_from_key_includes_guardrails_metadata(): assert "guardrails" in result["user_api_key_auth_metadata"] assert result["user_api_key_auth_metadata"]["guardrails"] == ["presidio", "aporia"] assert result["user_api_key_auth_metadata"]["other_field"] == "value" + + +@pytest.mark.asyncio +async def test_team_guardrails_append_to_key_guardrails(): + """ + Test that team guardrails are appended to key guardrails instead of overriding them. + Team guardrails should only be added if they are not already present in key guardrails. + """ + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + } + + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + metadata={"guardrails": ["key-guardrail-1", "key-guardrail-2"]}, + team_metadata={"guardrails": ["team-guardrail-1", "key-guardrail-1"]}, + ) + + with patch("litellm.proxy.utils._premium_user_check"): + updated_data = await add_litellm_data_to_request( + data=data, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + metadata = updated_data.get("metadata", {}) + guardrails = metadata.get("guardrails", []) + + assert "key-guardrail-1" in guardrails + assert "key-guardrail-2" in guardrails + assert "team-guardrail-1" in guardrails + assert guardrails.count("key-guardrail-1") == 1 + + +@pytest.mark.asyncio +async def test_request_guardrails_do_not_override_key_guardrails(): + """ + Test that request-level guardrails do not override key-level guardrails. + + Key guardrails should be preserved when request contains guardrails (including empty array). + """ + request_mock = MagicMock(spec=Request) + request_mock.url.path = "/chat/completions" + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/chat/completions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = {"Content-Type": "application/json"} + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + + user_api_key_dict = UserAPIKeyAuth( + api_key="test-key", + metadata={"guardrails": ["key-guardrail-1"]}, + team_metadata={}, + ) + + # Test case: Request with empty guardrails should not result in empty guardrails + data_with_empty = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "test"}], + "guardrails": [], + } + + with patch("litellm.proxy.utils._premium_user_check"): + updated_data_empty = await add_litellm_data_to_request( + data=data_with_empty, + request=request_mock, + user_api_key_dict=user_api_key_dict, + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + _metadata = updated_data_empty.get("metadata", {}) + requested_guardrails = _metadata.get("guardrails", []) + + assert "guardrails" not in updated_data_empty + assert "key-guardrail-1" in requested_guardrails + assert len(requested_guardrails) == 1 From 95dd216150a48df959bf409af29b178990bef911 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 28 Oct 2025 16:41:17 -0700 Subject: [PATCH 34/91] [UI] Feature - Add Apply Guardrail Testing Playground (#16030) * add applyGuardrail endpoints * v0 testing apply guard * fix: use tabs * move apply guardrails endpoint * fix apply_guardrail * fix applyGuardrail * fix apply guardrail for bedrock * test guard endpoints * add tooltip for enter button * refactor * add guardrail test * tests guardrails selector * TestNomaApplyGuardrail --- .../proxy/guardrails/endpoints.py | 43 --- .../proxy/guardrails/guardrail_endpoints.py | 38 +++ .../guardrails/guardrail_hooks/noma/noma.py | 90 ++++++- .../guardrails/guardrail_hooks/test_noma.py | 100 ++++++- .../guardrails/test_guardrail_endpoints.py | 111 ++++++-- .../src/components/guardrails.tsx | 110 +++++--- .../guardrails/GuardrailSelector.test.tsx | 65 +++++ .../guardrails/GuardrailTestPanel.test.tsx | 60 +++++ .../guardrails/GuardrailTestPanel.tsx | 169 ++++++++++++ .../GuardrailTestPlayground.test.tsx | 75 ++++++ .../guardrails/GuardrailTestPlayground.tsx | 254 ++++++++++++++++++ .../guardrails/GuardrailTestResults.test.tsx | 58 ++++ .../guardrails/GuardrailTestResults.tsx | 193 +++++++++++++ .../src/components/networking.tsx | 62 +++++ 14 files changed, 1322 insertions(+), 106 deletions(-) delete mode 100644 enterprise/litellm_enterprise/proxy/guardrails/endpoints.py create mode 100644 ui/litellm-dashboard/src/components/guardrails/GuardrailSelector.test.tsx create mode 100644 ui/litellm-dashboard/src/components/guardrails/GuardrailTestPanel.test.tsx create mode 100644 ui/litellm-dashboard/src/components/guardrails/GuardrailTestPanel.tsx create mode 100644 ui/litellm-dashboard/src/components/guardrails/GuardrailTestPlayground.test.tsx create mode 100644 ui/litellm-dashboard/src/components/guardrails/GuardrailTestPlayground.tsx create mode 100644 ui/litellm-dashboard/src/components/guardrails/GuardrailTestResults.test.tsx create mode 100644 ui/litellm-dashboard/src/components/guardrails/GuardrailTestResults.tsx diff --git a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py b/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py deleted file mode 100644 index 8b42b2549cd..00000000000 --- a/enterprise/litellm_enterprise/proxy/guardrails/endpoints.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Enterprise Guardrail Routes on LiteLLM Proxy - -To see all free guardrails see litellm/proxy/guardrails/* - - -Exposed Routes: -- /mask_pii -""" -from typing import Optional - -from fastapi import APIRouter, Depends - -from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.guardrails.guardrail_endpoints import GUARDRAIL_REGISTRY -from litellm.types.guardrails import ApplyGuardrailRequest, ApplyGuardrailResponse - -router = APIRouter(tags=["guardrails"], prefix="/guardrails") - - -@router.post("/apply_guardrail", response_model=ApplyGuardrailResponse) -async def apply_guardrail( - request: ApplyGuardrailRequest, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - Mask PII from a given text, requires a guardrail to be added to litellm. - """ - active_guardrail: Optional[ - CustomGuardrail - ] = GUARDRAIL_REGISTRY.get_initialized_guardrail_callback( - guardrail_name=request.guardrail_name - ) - if active_guardrail is None: - raise Exception(f"Guardrail {request.guardrail_name} not found") - - response_text = await active_guardrail.apply_guardrail( - text=request.text, language=request.language, entities=request.entities - ) - - return ApplyGuardrailResponse(response_text=response_text) diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 29da0ec40e6..f2f63778e44 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -10,10 +10,14 @@ from pydantic import BaseModel from litellm._logging import verbose_proxy_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.guardrails.guardrail_registry import GuardrailRegistry from litellm.types.guardrails import ( PII_ENTITY_CATEGORIES_MAP, + ApplyGuardrailRequest, + ApplyGuardrailResponse, BedrockGuardrailConfigModel, Guardrail, GuardrailEventHooks, @@ -1056,3 +1060,37 @@ async def get_provider_specific_params(): provider_params[guardrail_name] = fields return provider_params + +@router.post("/guardrails/apply_guardrail", response_model=ApplyGuardrailResponse) +@router.post("/apply_guardrail", response_model=ApplyGuardrailResponse) +async def apply_guardrail( + request: ApplyGuardrailRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Apply a guardrail to text input and return the processed result. + + This endpoint allows testing guardrails by applying them to custom text inputs. + """ + from litellm.proxy.utils import handle_exception_on_proxy + + try: + active_guardrail: Optional[ + CustomGuardrail + ] = GUARDRAIL_REGISTRY.get_initialized_guardrail_callback( + guardrail_name=request.guardrail_name + ) + if active_guardrail is None: + raise HTTPException( + status_code=404, + detail=f"Guardrail '{request.guardrail_name}' not found. Please ensure the guardrail is configured in your LiteLLM proxy.", + ) + + response_text = await active_guardrail.apply_guardrail( + text=request.text, language=request.language, entities=request.entities + ) + + return ApplyGuardrailResponse(response_text=response_text) + except Exception as e: + raise handle_exception_on_proxy(e) + diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py index 7073a4341fb..e049ca6a13c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma.py @@ -9,7 +9,7 @@ import asyncio import copy import os from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, Final, Literal, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Final, List, Literal, Optional, Type, Union from urllib.parse import urljoin from fastapi import HTTPException @@ -23,7 +23,7 @@ from litellm.llms.custom_httpx.http_handler import ( httpxSpecialProvider, ) from litellm.proxy._types import UserAPIKeyAuth -from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.guardrails import GuardrailEventHooks, PiiEntityType from litellm.types.utils import EmbeddingResponse, GuardrailStatus, ImageResponse # Constants @@ -851,6 +851,92 @@ class NomaGuardrail(CustomGuardrail): else: verbose_proxy_logger.debug(msg) + async def apply_guardrail( + self, + text: str, + language: Optional[str] = None, + entities: Optional[List[PiiEntityType]] = None, + ) -> str: + """ + Apply Noma guardrail to the given text for testing purposes. + + This method allows users to test Noma guardrails without making actual LLM calls. + It creates a mock request to test the guardrail functionality. + + Args: + text: The text to analyze + language: Optional language parameter (not used by Noma) + entities: Optional entities parameter (not used by Noma) + + Returns: + The original text if allowed, or anonymized text if available + + Raises: + Exception: If the content is blocked by Noma guardrail + """ + try: + verbose_proxy_logger.debug("Noma Guardrail: Applying guardrail") + + # Create a mock user auth object for testing + from litellm.proxy._types import UserAPIKeyAuth + mock_user_auth = UserAPIKeyAuth() + + # Create payload for Noma API + payload = {"request": {"text": text}} + + # Call Noma API + response_json = await self._call_noma_api( + payload=payload, + llm_request_id=None, + request_data={"messages": [{"role": "user", "content": text}]}, + user_auth=mock_user_auth, + extra_data={}, + ) + + # Check if content is blocked + verdict = response_json.get("verdict", True) + if not verdict: + # Check if we should anonymize instead of blocking + if self.anonymize_input and self._should_anonymize(response_json, USER_ROLE): + anonymized_content = self._extract_anonymized_content( + response_json, USER_ROLE + ) + if anonymized_content: + verbose_proxy_logger.debug( + "Noma Guardrail: Content anonymized" + ) + return anonymized_content + + # Content is blocked + original_response = response_json.get("originalResponse", {}) + filtered_response = NomaBlockedMessage(original_response)._filter_triggered_classifications(original_response) + raise Exception( + f"Content blocked by Noma guardrail: {filtered_response}" + ) + + # Check if anonymization is available even for allowed content + if self.anonymize_input: + anonymized_content = self._extract_anonymized_content( + response_json, USER_ROLE + ) + if anonymized_content: + verbose_proxy_logger.debug( + "Noma Guardrail: Content anonymized" + ) + return anonymized_content + + verbose_proxy_logger.debug( + "Noma Guardrail: Successfully applied guardrail" + ) + + return text + + except Exception as e: + verbose_proxy_logger.error( + "Noma Guardrail: Failed to apply guardrail: %s", str(e) + ) + raise Exception(f"Noma guardrail failed: {str(e)}") + @staticmethod def get_config_model() -> Optional[Type["GuardrailConfigModel"]]: from litellm.types.proxy.guardrails.guardrail_hooks.noma import ( diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma.py index d98473a1280..f9b80a0a579 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_noma.py @@ -731,7 +731,7 @@ class TestBackgroundProcessing: ): """Test post-call success hook in monitor mode""" from litellm.types.utils import Choices, Message - + # Update event hook to post_call monitor_mode_guardrail.event_hook = "post_call" @@ -764,6 +764,104 @@ class TestBackgroundProcessing: mock_create_background.assert_called_once() +class TestNomaApplyGuardrail: + """ + Test the apply_guardrail method for Noma guardrails + """ + + @pytest.mark.asyncio + async def test_apply_guardrail_success(self): + """ + Test that apply_guardrail returns text when content is allowed + """ + guardrail = NomaGuardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + application_id="test-app", + monitor_mode=False, + block_failures=True, + ) + + mock_response = MagicMock() + mock_response.json.return_value = {"verdict": True} + mock_response.raise_for_status = MagicMock() + + with patch.object( + guardrail.async_handler, "post", return_value=mock_response + ): + result = await guardrail.apply_guardrail( + text="This is a safe test message" + ) + + assert result == "This is a safe test message" + + @pytest.mark.asyncio + async def test_apply_guardrail_blocked(self): + """ + Test that apply_guardrail raises exception when content is blocked + """ + guardrail = NomaGuardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + application_id="test-app", + monitor_mode=False, + block_failures=True, + ) + + mock_response = MagicMock() + mock_response.json.return_value = { + "verdict": False, + "originalResponse": { + "prompt": {"contentDetector": {"result": True, "confidence": 0.9}} + }, + } + mock_response.raise_for_status = MagicMock() + + with patch.object( + guardrail.async_handler, "post", return_value=mock_response + ): + with pytest.raises(Exception) as exc_info: + await guardrail.apply_guardrail(text="This is blocked content") + + assert "Content blocked by Noma guardrail" in str(exc_info.value) + + @pytest.mark.asyncio + async def test_apply_guardrail_with_anonymization(self): + """ + Test that apply_guardrail returns anonymized text when anonymize_input is enabled + """ + guardrail = NomaGuardrail( + api_key="test-api-key", + api_base="https://api.test.noma.security/", + application_id="test-app", + anonymize_input=True, + monitor_mode=False, + block_failures=True, + ) + + mock_response = MagicMock() + mock_response.json.return_value = { + "verdict": True, + "originalResponse": { + "prompt": { + "anonymizedContent": { + "anonymized": "My email is ******* and phone is *******" + } + } + }, + } + mock_response.raise_for_status = MagicMock() + + with patch.object( + guardrail.async_handler, "post", return_value=mock_response + ): + result = await guardrail.apply_guardrail( + text="My email is test@example.com and phone is 123-456-7890" + ) + + assert result == "My email is ******* and phone is *******" + + class TestIntegration: @pytest.mark.asyncio async def test_full_guardrail_flow(self): diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index 34a1ae4b385..75daf540165 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -14,25 +14,27 @@ sys.path.insert( from fastapi import HTTPException from litellm.proxy.guardrails.guardrail_endpoints import ( + CreateGuardrailRequest, + PatchGuardrailRequest, + UpdateGuardrailRequest, + apply_guardrail, + create_guardrail, + delete_guardrail, get_guardrail_info, list_guardrails_v2, - CreateGuardrailRequest, - create_guardrail, - UpdateGuardrailRequest, - update_guardrail, - PatchGuardrailRequest, patch_guardrail, - delete_guardrail, + update_guardrail, ) from litellm.proxy.guardrails.guardrail_registry import ( IN_MEMORY_GUARDRAIL_HANDLER, InMemoryGuardrailHandler, ) from litellm.types.guardrails import ( + ApplyGuardrailRequest, BaseLitellmParams, + Guardrail, GuardrailInfoResponse, LitellmParams, - Guardrail, ) # Mock data for testing @@ -343,8 +345,11 @@ def test_optional_params_returned_when_properly_overridden(): async def test_bedrock_guardrail_prepare_request_with_api_key(): """Test _prepare_request method uses Bearer token when api_key is provided in data""" from unittest.mock import Mock, patch - from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail - + + from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( + BedrockGuardrail, + ) + # Setup guardrail hook guardrail_hook = BedrockGuardrail( guardrailIdentifier="test-guardrail-id", @@ -377,8 +382,11 @@ async def test_bedrock_guardrail_prepare_request_with_api_key(): async def test_bedrock_guardrail_prepare_request_without_api_key(): """Test _prepare_request method falls back to SigV4 when no api_key is provided""" from unittest.mock import Mock, patch - from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail - + + from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( + BedrockGuardrail, + ) + # Setup guardrail hook guardrail_hook = BedrockGuardrail( guardrailIdentifier="test-guardrail-id", @@ -427,8 +435,11 @@ async def test_bedrock_guardrail_prepare_request_without_api_key(): async def test_bedrock_guardrail_prepare_request_with_bearer_token_env(): """Test _prepare_request method uses Bearer token from environment when available""" from unittest.mock import Mock, patch - from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail - + + from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( + BedrockGuardrail, + ) + # Setup guardrail hook guardrail_hook = BedrockGuardrail( guardrailIdentifier="test-guardrail-id", @@ -469,8 +480,11 @@ async def test_bedrock_guardrail_prepare_request_with_bearer_token_env(): @pytest.mark.asyncio async def test_bedrock_guardrail_make_api_request_passes_api_key(): """Test make_bedrock_api_request method correctly passes api_key from request_data""" - from unittest.mock import Mock, patch, AsyncMock - from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import BedrockGuardrail + from unittest.mock import AsyncMock, Mock, patch + + from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( + BedrockGuardrail, + ) guardrail_hook = BedrockGuardrail( guardrailIdentifier="test-guardrail-id", @@ -850,4 +864,69 @@ async def test_delete_guardrail_endpoint( if scenario == "success_sync_fails": assert mock_logger is not None mock_logger.warning.assert_called_once() - assert "Failed to remove guardrail" in str(mock_logger.warning.call_args) \ No newline at end of file + assert "Failed to remove guardrail" in str(mock_logger.warning.call_args) + + +@pytest.mark.asyncio +async def test_apply_guardrail_not_found(mocker): + """ + Test apply_guardrail endpoint returns proper error when guardrail is not found. + """ + from litellm.proxy._types import ProxyException, UserAPIKeyAuth + + # Mock the GUARDRAIL_REGISTRY to return None (guardrail not found) + mock_registry = mocker.Mock() + mock_registry.get_initialized_guardrail_callback.return_value = None + mocker.patch("litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", mock_registry) + + # Create request + request = ApplyGuardrailRequest( + guardrail_name="non-existent-guardrail", + text="Test input text" + ) + + # Mock user auth + mock_user_auth = UserAPIKeyAuth() + + # Call endpoint and expect ProxyException + with pytest.raises(ProxyException) as exc_info: + await apply_guardrail(request=request, user_api_key_dict=mock_user_auth) + + # Verify error details + assert str(exc_info.value.code) == "404" + assert "not found" in str(exc_info.value.message).lower() + + +@pytest.mark.asyncio +async def test_apply_guardrail_execution_error(mocker): + """ + Test apply_guardrail endpoint handles exceptions from guardrail execution properly. + """ + from litellm.proxy._types import ProxyException, UserAPIKeyAuth + + # Mock guardrail that raises an exception + mock_guardrail = mocker.Mock() + mock_guardrail.apply_guardrail = AsyncMock( + side_effect=Exception("Bedrock guardrail failed: Violated guardrail policy") + ) + + # Mock the GUARDRAIL_REGISTRY + mock_registry = mocker.Mock() + mock_registry.get_initialized_guardrail_callback.return_value = mock_guardrail + mocker.patch("litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", mock_registry) + + # Create request + request = ApplyGuardrailRequest( + guardrail_name="test-guardrail", + text="Test input text with forbidden content" + ) + + # Mock user auth + mock_user_auth = UserAPIKeyAuth() + + # Call endpoint and expect ProxyException + with pytest.raises(ProxyException) as exc_info: + await apply_guardrail(request=request, user_api_key_dict=mock_user_auth) + + # Verify error is properly handled + assert "Bedrock guardrail failed" in str(exc_info.value.message) \ No newline at end of file diff --git a/ui/litellm-dashboard/src/components/guardrails.tsx b/ui/litellm-dashboard/src/components/guardrails.tsx index 605770831d7..23aec34c404 100644 --- a/ui/litellm-dashboard/src/components/guardrails.tsx +++ b/ui/litellm-dashboard/src/components/guardrails.tsx @@ -1,11 +1,12 @@ import React, { useState, useEffect } from "react"; -import { Button } from "@tremor/react"; +import { Button, TabGroup, TabList, Tab, TabPanels, TabPanel } from "@tremor/react"; import { Modal } from "antd"; import { getGuardrailsList, deleteGuardrailCall } from "./networking"; import AddGuardrailForm from "./guardrails/add_guardrail_form"; import GuardrailTable from "./guardrails/guardrail_table"; import { isAdminRole } from "@/utils/roles"; import GuardrailInfoView from "./guardrails/guardrail_info"; +import GuardrailTestPlayground from "./guardrails/GuardrailTestPlayground"; import NotificationsManager from "./molecules/notifications_manager"; interface GuardrailsPanelProps { @@ -37,6 +38,7 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole const [isDeleting, setIsDeleting] = useState(false); const [guardrailToDelete, setGuardrailToDelete] = useState<{ id: string; name: string } | null>(null); const [selectedGuardrailId, setSelectedGuardrailId] = useState(null); + const [activeTab, setActiveTab] = useState(0); const isAdmin = userRole ? isAdminRole(userRole) : false; @@ -104,52 +106,72 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole return (
-
- -
+ + + Guardrails + Test Playground + - {selectedGuardrailId ? ( - setSelectedGuardrailId(null)} - accessToken={accessToken} - isAdmin={isAdmin} - /> - ) : ( - setSelectedGuardrailId(id)} - /> - )} + + +
+ +
- + {selectedGuardrailId ? ( + setSelectedGuardrailId(null)} + accessToken={accessToken} + isAdmin={isAdmin} + /> + ) : ( + setSelectedGuardrailId(id)} + /> + )} - {guardrailToDelete && ( - -

Are you sure you want to delete guardrail: {guardrailToDelete.name} ?

-

This action cannot be undone.

-
- )} + + + {guardrailToDelete && ( + +

Are you sure you want to delete guardrail: {guardrailToDelete.name} ?

+

This action cannot be undone.

+
+ )} +
+ + + setActiveTab(0)} + /> + +
+
); }; diff --git a/ui/litellm-dashboard/src/components/guardrails/GuardrailSelector.test.tsx b/ui/litellm-dashboard/src/components/guardrails/GuardrailSelector.test.tsx new file mode 100644 index 00000000000..9da2b09b51e --- /dev/null +++ b/ui/litellm-dashboard/src/components/guardrails/GuardrailSelector.test.tsx @@ -0,0 +1,65 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import GuardrailSelector from "./GuardrailSelector"; +import * as networking from "../networking"; + +vi.mock("../networking"); + +Object.defineProperty(window, "matchMedia", { + writable: true, + value: vi.fn().mockImplementation((query) => ({ + matches: false, + media: query, + onchange: null, + addListener: vi.fn(), + removeListener: vi.fn(), + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + dispatchEvent: vi.fn(), + })), +}); + +describe("GuardrailSelector", () => { + const mockAccessToken = "test-token"; + const mockOnChange = vi.fn(); + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should load guardrails from API when component mounts", async () => { + /** + * Tests that the selector fetches guardrails from the API on mount. + * This validates the core data loading functionality. + */ + const mockGuardrails = [ + { + guardrail_name: "pii-guard", + litellm_params: { guardrail: "presidio", mode: "pre_call", default_on: false }, + }, + { + guardrail_name: "content-filter", + litellm_params: { guardrail: "lakera", mode: "post_call", default_on: true }, + }, + ]; + + vi.mocked(networking.getGuardrailsList).mockResolvedValue({ + guardrails: mockGuardrails, + }); + + render( + + ); + + // Verify API was called with correct token + await waitFor(() => { + expect(networking.getGuardrailsList).toHaveBeenCalledWith(mockAccessToken); + }); + }); +}); + diff --git a/ui/litellm-dashboard/src/components/guardrails/GuardrailTestPanel.test.tsx b/ui/litellm-dashboard/src/components/guardrails/GuardrailTestPanel.test.tsx new file mode 100644 index 00000000000..fe99baf0935 --- /dev/null +++ b/ui/litellm-dashboard/src/components/guardrails/GuardrailTestPanel.test.tsx @@ -0,0 +1,60 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { GuardrailTestPanel } from "./GuardrailTestPanel"; + +Object.defineProperty(window, "matchMedia", { + writable: true, + value: vi.fn().mockImplementation((query) => ({ + matches: false, + media: query, + onchange: null, + addListener: vi.fn(), + removeListener: vi.fn(), + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + dispatchEvent: vi.fn(), + })), +}); + +describe("GuardrailTestPanel", () => { + const mockOnSubmit = vi.fn(); + const mockOnClose = vi.fn(); + const mockGuardrailNames = ["test-guardrail-1", "test-guardrail-2"]; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should submit text when Enter key is pressed", async () => { + /** + * Tests that pressing Enter submits the form with the input text. + * This validates the keyboard shortcut functionality. + */ + const user = userEvent.setup(); + + render( + + ); + + // Find and type in the textarea + const textarea = screen.getByPlaceholderText("Enter text to test with guardrails..."); + await user.type(textarea, "Test input text"); + + // Press Enter to submit + await user.keyboard("{Enter}"); + + // Verify onSubmit was called with the correct text + await waitFor(() => { + expect(mockOnSubmit).toHaveBeenCalledWith("Test input text"); + }); + }); +}); + diff --git a/ui/litellm-dashboard/src/components/guardrails/GuardrailTestPanel.tsx b/ui/litellm-dashboard/src/components/guardrails/GuardrailTestPanel.tsx new file mode 100644 index 00000000000..8cda73790c6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/guardrails/GuardrailTestPanel.tsx @@ -0,0 +1,169 @@ +import React, { useState } from "react"; +import { Button } from "@tremor/react"; +import { Input, Typography, Tooltip } from "antd"; +import { CopyOutlined, InfoCircleOutlined } from "@ant-design/icons"; +import NotificationsManager from "../molecules/notifications_manager"; +import GuardrailTestResults from "./GuardrailTestResults"; + +const { TextArea } = Input; +const { Text } = Typography; + +interface GuardrailTestPanelProps { + guardrailNames: string[]; + onSubmit: (text: string) => void; + isLoading: boolean; + results: Array<{ guardrailName: string; response_text: string; latency: number }> | null; + errors: Array<{ guardrailName: string; error: Error; latency: number }> | null; + onClose: () => void; +} + +export function GuardrailTestPanel({ + guardrailNames, + onSubmit, + isLoading, + results, + errors, + onClose, +}: GuardrailTestPanelProps) { + const [inputText, setInputText] = useState(""); + + const handleSubmit = () => { + if (!inputText.trim()) { + NotificationsManager.fromBackend("Please enter text to test"); + return; + } + + onSubmit(inputText); + }; + + const handleKeyDown = (e: React.KeyboardEvent) => { + if (e.key === "Enter" && !e.shiftKey && !e.ctrlKey && !e.metaKey) { + e.preventDefault(); + handleSubmit(); + } + }; + + const copyToClipboard = async (text: string) => { + try { + if (navigator.clipboard && window.isSecureContext) { + await navigator.clipboard.writeText(text); + return true; + } else { + const textArea = document.createElement("textarea"); + textArea.value = text; + textArea.style.position = "fixed"; + textArea.style.opacity = "0"; + document.body.appendChild(textArea); + textArea.focus(); + textArea.select(); + + const successful = document.execCommand("copy"); + document.body.removeChild(textArea); + + if (!successful) { + throw new Error("execCommand failed"); + } + return true; + } + } catch (error) { + console.error("Copy failed:", error); + return false; + } + }; + + const handleCopyInput = async () => { + const success = await copyToClipboard(inputText); + if (success) { + NotificationsManager.success("Input copied to clipboard"); + } else { + NotificationsManager.fromBackend("Failed to copy input"); + } + }; + + return ( +
+ {/* Header */} +
+
+
+
+

Test Guardrails:

+
+ {guardrailNames.map((name) => ( +
+ {name} +
+ ))} +
+
+

+ Test {guardrailNames.length > 1 ? "guardrails" : "guardrail"} and compare results +

+
+
+
+ + {/* Input Section */} +
+
+
+
+
+ + + + +
+ {inputText && ( + + )} +
+