From 8d2355392a82dd8cd5640e69c01b4513cdacf279 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 08:18:30 +0000 Subject: [PATCH 01/20] feat(guardrails): add logging_only_scope to observe one direction Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 27 +++- .../proxy/guardrails/guardrail_registry.py | 15 +++ litellm/types/guardrails.py | 10 ++ .../observability/test_guardrail_effects.py | 94 ++++++++++++++ .../guardrails/test_guardrail_registry.py | 118 +++++++++++++++++- .../integrations/test_custom_guardrail.py | 49 ++++++-- 6 files changed, 293 insertions(+), 20 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 2eb9cfb5042..ffe9544007c 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -21,6 +21,7 @@ from litellm.types.guardrails import ( DynamicGuardrailParams, GuardrailEventHooks, LitellmParams, + LoggingOnlyScope, Mode, ) from litellm.types.llms.openai import AllMessageValues @@ -175,6 +176,7 @@ class CustomGuardrail(CustomLogger): use_native_lifecycle_hooks: ClassVar[bool] = False records_own_guardrail_information: ClassVar[bool] = False + logging_only_scope: LoggingOnlyScope | None def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: forwarded to cooperative __init_subclass__ hooks super().__init_subclass__(**kwargs) @@ -246,6 +248,7 @@ class CustomGuardrail(CustomLogger): self.run_in_parallel: bool = run_in_parallel self.scan_raw_request: bool = scan_raw_request self.only_scan_new_messages: bool = only_scan_new_messages + self.logging_only_scope = None if supported_event_hooks: ## validate event_hook is in supported_event_hooks @@ -803,6 +806,13 @@ class CustomGuardrail(CustomLogger): def uses_apply_guardrail_interface(self) -> bool: return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail + def supports_logging_only_scope(self) -> bool: + return ( + self.uses_apply_guardrail_interface() + and not self.use_native_lifecycle_hooks + and type(self).async_logging_hook is CustomGuardrail.async_logging_hook + ) + def _deployment_hook_target(self) -> "CustomLogger": if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks: return self @@ -999,12 +1009,19 @@ class CustomGuardrail(CustomLogger): "litellm_call_id": kwargs.get("litellm_call_id"), "metadata": scratch_metadata, } - await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self) - if response is None: + if self.logging_only_scope != "output": + try: + await translation.process_input_messages(data=scratch_request, guardrail_to_apply=self) + except Exception as e: + verbose_logger.warning("Guardrail %s: logging_only scan raised: %s", self.guardrail_name, e) + if response is None or self.logging_only_scope == "input": return - await output_translation.process_output_response( - response=copy.deepcopy(response), guardrail_to_apply=self, request_data=scratch_request - ) + try: + await output_translation.process_output_response( + response=copy.deepcopy(response), guardrail_to_apply=self, request_data=scratch_request + ) + except Exception as e: + verbose_logger.warning("Guardrail %s: logging_only scan raised: %s", self.guardrail_name, e) def supports_scan_only_tool_results(self) -> bool: """Whether this guardrail can scan tool-result content. diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 0dc50cd6196..0956d2c9cfc 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -54,6 +54,7 @@ from .guardrail_hooks.llm_as_a_judge import ( initialize_guardrail as initialize_llm_as_a_judge, ) from .guardrail_initializers import ( + _configured_event_hooks, initialize_bedrock, initialize_hide_secrets, initialize_lakera, @@ -439,6 +440,20 @@ def _as_callback_tuple( def _configure_callback_scoping( custom_guardrail_callback: CustomGuardrail, guardrail_name: str, litellm_params: LitellmParams ) -> None: + custom_guardrail_callback.logging_only_scope = litellm_params.logging_only_scope + logging_only_scope: Final = litellm_params.logging_only_scope + if logging_only_scope is not None and GuardrailEventHooks.logging_only.value not in _configured_event_hooks( + litellm_params.mode + ): + raise ValueError( + f"Guardrail {guardrail_name}: logging_only_scope is set, but mode does not include logging_only, " + "so it would never apply. Add logging_only to mode or remove logging_only_scope." + ) + if logging_only_scope in ("input", "output") and not custom_guardrail_callback.supports_logging_only_scope(): + raise ValueError( + f"Guardrail {guardrail_name}: logging_only_scope={logging_only_scope!r} is not supported by this " + "guardrail, whose logging_only hook scans on its own. Remove logging_only_scope." + ) for scoping_param in ( "skip_system_message_in_guardrail", "skip_tool_message_in_guardrail", diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 579a3f6322f..a98446fff43 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -895,6 +895,8 @@ class ContentFilterConfigModel(BaseModel): MCP_SECURITY_ON_VIOLATION: Final = frozenset({"block", "alert"}) +LoggingOnlyScope = Literal["input", "output", "both"] + class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch update guardrails api_key: str | None = Field(default=None, description="API key for the guardrail service") @@ -1136,6 +1138,14 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up ), ) + logging_only_scope: LoggingOnlyScope | None = Field( + default=None, + description=( + "which direction a logging_only scan observes: 'input' (request), 'output' (response), or 'both' " + "(default). Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking." + ), + ) + @field_validator( "mode", "default_action", diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index c448473391f..4e9691bd4e9 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -4,6 +4,7 @@ import signal import socket import uuid from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone from pathlib import Path from typing import Final @@ -1297,3 +1298,96 @@ def test_responses_pre_call_denial_stream_survives_worker_kill(gateway: Gateway, for response in responses: assert response.status_code == 200, response.text assert response.headers["content-type"].startswith("text/event-stream"), response.text + + +@pytest.mark.parametrize( + ("logging_only_scope", "scanned_directions"), + (("input", ("request",)), ("output", ("response",)), ("both", ("request", "response"))), +) +def test_logging_only_scope_observes_only_the_configured_direction_without_blocking( + gateway: Gateway, tmp_path: Path, logging_only_scope: str, scanned_directions: tuple[str, ...] +) -> None: + identity: Final = "guardrail" + uuid.uuid4().hex + prompt: Final = "synthetic observed prompt " + identity + reply: Final = "synthetic observed reply " + identity + texts_by_direction: Final = {"request": [prompt], "response": [reply]} + + def guardrail(request: Request) -> Reply: + assert request.target == "/beta/litellm_basic_guardrail_api" + return Reply(body=json.dumps({"action": "BLOCKED", "blocked_reason": "synthetic observed denial"}).encode()) + + def provider(request: Request) -> Reply: + assert request.target == "/v1/chat/completions" + assert json.loads(request.body)["messages"] == [{"role": "user", "content": prompt}] + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": reply}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14}, + } + ).encode() + ) + + with wire_server(guardrail) as policy, wire_server(provider) as upstream: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["guardrails"] = [ + { + "guardrail_name": identity, + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "logging_only", + "logging_only_scope": logging_only_scope, + "default_on": True, + "api_base": policy.url, + "api_key": "synthetic-guardrail-key", + }, + } + ] + path: Final = tmp_path / "logging-only-scope.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario: + model: Final = scenario.model(api_base=upstream.url + "/v1") + response: Final = candidate.request( + "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": prompt}]} + ) + assert response.status_code == 200, response.text + assert response.json()["choices"][0]["message"]["content"] == reply, response.text + assert len(upstream.drain()) == 1 + rows: Final = eventually( + lambda: read_rows('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda values: len(values) == 1, + seconds=70, + ) + scans: Final = tuple(json.loads(scan.body) for scan in policy.drain()) + assert [(scan["input_type"], scan["texts"]) for scan in scans] == [ + (direction, texts_by_direction[direction]) for direction in scanned_directions + ], scans + entries: Final = object_value(rows[0]["metadata"])["guardrail_information"] + assert isinstance(entries, list), rows[0] + assert [ + (entry["guardrail_name"], entry["guardrail_mode"], entry["guardrail_status"]) + for entry in map(object_value, entries) + ] == [(identity, "logging_only", "guardrail_intervened")] * len(scanned_directions), entries + today: Final = datetime.now(timezone.utc).date().isoformat() + guardrail_id: Final = next( + object_value(row)["guardrail_id"] + for row in candidate.get("/v2/guardrails/list")["guardrails"] + if object_value(row)["guardrail_name"] == identity + ) + detail: Final = eventually( + lambda: candidate.request( + "GET", + f"/guardrails/usage/detail/{guardrail_id}", + params={"start_date": today, "end_date": today}, + ).json(), + lambda body: body["requestsEvaluated"] >= len(scanned_directions), + seconds=30, + return_last_on_timeout=True, + ) + assert detail["requestsEvaluated"] == len(scanned_directions), detail diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 022fe85c779..d8c6351a384 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -1,15 +1,18 @@ from collections.abc import Iterable +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from pydantic import ValidationError from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy.guardrails.guardrail_registry import ( - get_guardrail_initializer_from_hooks, GuardrailRegistry, InMemoryGuardrailHandler, + get_guardrail_initializer_from_hooks, ) -from litellm.types.guardrails import GuardrailEventHooks, Guardrail, LitellmParams +from litellm.types.guardrails import Guardrail, GuardrailEventHooks, LitellmParams, LoggingOnlyScope, Mode +from litellm.types.utils import GenericGuardrailAPIInputs def test_get_guardrail_initializer_from_hooks(): @@ -932,6 +935,117 @@ class TestScanOnlyToolResultsInitRefusal: ) +class _LoggingOnlyScopeSupportedGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: str, + logging_obj: object | None = None, + ) -> GenericGuardrailAPIInputs: + return inputs + + +class _LoggingOnlyScopeUnsupportedGuardrail(_LoggingOnlyScopeSupportedGuardrail): + async def async_logging_hook( + self, + kwargs: dict[str, object], + result: object, + call_type: str, + ) -> tuple[dict[str, object], object]: + return kwargs, result + + +class TestLoggingOnlyScopeValidation: + def _initialize( + self, + mode: str | list[str] | Mode, + scope: LoggingOnlyScope | None, + callback_type: type[CustomGuardrail] = _LoggingOnlyScopeSupportedGuardrail, + ) -> CustomGuardrail: + from litellm.proxy.guardrails import guardrail_registry as registry_module + + guardrail_type: Final = "logging_only_scope_test" + + def _initializer(litellm_params: LitellmParams, guardrail: Guardrail) -> CustomGuardrail: + return callback_type( + guardrail_name=guardrail["guardrail_name"], + event_hook=litellm_params.mode, + ) + + registry_module.guardrail_initializer_registry[guardrail_type] = _initializer + lists: Final = _all_callback_lists() + snapshots: Final = [list(callback_list) for callback_list in lists] + try: + handler: Final = InMemoryGuardrailHandler() + result: Final = handler.initialize_guardrail( + guardrail={ + "guardrail_name": "logging-only-scope-guardrail", + "litellm_params": { + "guardrail": guardrail_type, + "mode": mode, + "logging_only_scope": scope, + }, + } + ) + assert result is not None + callback: Final = handler.guardrail_id_to_custom_guardrail[result["guardrail_id"]] + assert callback is not None + return callback + finally: + for callback_list, snapshot in zip(lists, snapshots): + callback_list[:] = snapshot + registry_module.guardrail_initializer_registry.pop(guardrail_type, None) + + def test_scope_requires_logging_only_mode(self) -> None: + with pytest.raises(ValueError, match="logging_only_scope is set") as exc_info: + self._initialize(mode="pre_call", scope="input") + + assert str(exc_info.value) == ( + "Guardrail logging-only-scope-guardrail: logging_only_scope is set, but mode does not include " + "logging_only, so it would never apply. Add logging_only to mode or remove logging_only_scope." + ) + + @pytest.mark.parametrize( + "mode", + ( + "logging_only", + ["pre_call", "logging_only"], + Mode(tags={"audit": "logging_only"}, default="pre_call"), + ), + ) + def test_scope_accepts_logging_only_in_supported_mode_forms(self, mode: str | list[str] | Mode) -> None: + callback: Final = self._initialize(mode=mode, scope="input") + + assert callback.logging_only_scope == "input" + + def test_directional_scope_rejected_when_guardrail_owns_logging_hook(self) -> None: + with pytest.raises(ValueError, match="logging_only_scope='input' is not supported") as exc_info: + self._initialize( + mode="logging_only", + scope="input", + callback_type=_LoggingOnlyScopeUnsupportedGuardrail, + ) + + assert str(exc_info.value) == ( + "Guardrail logging-only-scope-guardrail: logging_only_scope='input' is not supported by this " + "guardrail, whose logging_only hook scans on its own. Remove logging_only_scope." + ) + + def test_both_scope_accepted_when_guardrail_owns_logging_hook(self) -> None: + callback: Final = self._initialize( + mode="logging_only", + scope="both", + callback_type=_LoggingOnlyScopeUnsupportedGuardrail, + ) + + assert callback.logging_only_scope == "both" + + def test_invalid_scope_fails_litellm_params_validation(self) -> None: + with pytest.raises(ValidationError): + LitellmParams(guardrail="test", mode="logging_only", logging_only_scope="request") + + @pytest.mark.asyncio async def test_update_guardrail_in_db_raises_when_row_missing(): prisma_client = MagicMock() diff --git a/tests/unit/integrations/test_custom_guardrail.py b/tests/unit/integrations/test_custom_guardrail.py index 4649bddd281..2fdb338f20e 100644 --- a/tests/unit/integrations/test_custom_guardrail.py +++ b/tests/unit/integrations/test_custom_guardrail.py @@ -12,7 +12,7 @@ from litellm.integrations.custom_guardrail import ( ) from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy._types import CallTypes, UserAPIKeyAuth -from litellm.types.guardrails import GuardrailEventHooks, Mode +from litellm.types.guardrails import GuardrailEventHooks, LoggingOnlyScope, Mode from litellm.types.utils import ( Choices, GenericGuardrailAPIInputs, @@ -546,14 +546,10 @@ class TestApplyGuardrailCheck: class ParentGuardrail(CustomGuardrail): """Parent that inherits apply_guardrail from CustomGuardrail""" - pass - # Child class that only inherits apply_guardrail (doesn't override) class ChildGuardrailWithoutOverride(ParentGuardrail): """Child that only inherits apply_guardrail""" - pass - # Child class that overrides apply_guardrail class ChildGuardrailWithOverride(ParentGuardrail): """Child that overrides apply_guardrail""" @@ -2540,7 +2536,7 @@ class TestLoggingOnlyApplyGuardrail: from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( ContentFilterGuardrail, ) - from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks + from litellm.types.guardrails import BlockedWord, ContentFilterAction guardrail: Final = ContentFilterGuardrail( guardrail_name="content-review", @@ -2577,6 +2573,32 @@ class TestLoggingOnlyApplyGuardrail: assert "standard_logging_guardrail_information" not in kwargs["litellm_params"]["metadata"] assert kwargs["standard_logging_object"] == {"guardrail_information": None} + @pytest.mark.parametrize( + "scope,expected_calls", + ( + (None, [("request", ["hello there"]), ("response", ["general kenobi"])]), + ("both", [("request", ["hello there"]), ("response", ["general kenobi"])]), + ("input", [("request", ["hello there"])]), + ("output", [("response", ["general kenobi"])]), + ), + ) + @pytest.mark.asyncio + async def test_logging_only_scope_scans_configured_directions( + self, + scope: LoggingOnlyScope | None, + expected_calls: list[tuple[str, list[str]]], + ) -> None: + guardrail: Final = _ApplyOnlyObserver() + guardrail.logging_only_scope = scope + kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == expected_calls + entries: Final = out_kwargs["standard_logging_object"]["guardrail_information"] + assert len(entries) == len(expected_calls) + assert [entry["guardrail_mode"] for entry in entries] == ["logging_only"] * len(expected_calls) + @pytest.mark.asyncio async def test_appends_to_pre_call_verdicts_without_duplicating_them(self): guardrail = _ApplyOnlyObserver() @@ -2606,14 +2628,17 @@ class TestLoggingOnlyApplyGuardrail: @pytest.mark.asyncio async def test_block_verdict_is_recorded_without_raising(self): - guardrail = _ApplyOnlyObserver(block=True) + guardrail: Final = _ApplyOnlyObserver(block=True) kwargs, response = _logged_call([{"role": "user", "content": "flagged content"}]) out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) - assert guardrail.calls == [("request", ["flagged content"])] - entries = out_kwargs["standard_logging_object"]["guardrail_information"] - assert [e["guardrail_status"] for e in entries] == ["guardrail_intervened"] + assert guardrail.calls == [ + ("request", ["flagged content"]), + ("response", ["general kenobi"]), + ] + entries: Final = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [entry["guardrail_status"] for entry in entries] == ["guardrail_intervened", "guardrail_intervened"] @pytest.mark.asyncio async def test_call_type_without_translation_is_skipped(self): @@ -2939,9 +2964,7 @@ async def test_native_lifecycle_guardrail_logging_only_scans_assembled_response( from litellm.types.utils import Choices, Message, ModelResponse guardrail = _NativeLifecycleLoggingGuardrail() - assembled = ModelResponse( - choices=[Choices(message=Message(role="assistant", content="assembled stream text"))] - ) + assembled = ModelResponse(choices=[Choices(message=Message(role="assistant", content="assembled stream text"))]) sentinel_result = object() kwargs = { "model": "gpt-5.4-mini", From c63fd3bd29b48506dabb930624b399294e022a38 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 08:21:00 +0000 Subject: [PATCH 02/20] chore(guardrails): remove unrelated test churn Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/guardrails/guardrail_registry.py | 2 +- .../proxy/guardrails/test_guardrail_registry.py | 4 ++-- tests/unit/integrations/test_custom_guardrail.py | 14 ++++++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 0956d2c9cfc..6527f6be50b 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -440,8 +440,8 @@ def _as_callback_tuple( def _configure_callback_scoping( custom_guardrail_callback: CustomGuardrail, guardrail_name: str, litellm_params: LitellmParams ) -> None: - custom_guardrail_callback.logging_only_scope = litellm_params.logging_only_scope logging_only_scope: Final = litellm_params.logging_only_scope + custom_guardrail_callback.logging_only_scope = logging_only_scope if logging_only_scope is not None and GuardrailEventHooks.logging_only.value not in _configured_event_hooks( litellm_params.mode ): diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index d8c6351a384..6af7a7a06a4 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -7,11 +7,11 @@ from pydantic import ValidationError from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy.guardrails.guardrail_registry import ( + get_guardrail_initializer_from_hooks, GuardrailRegistry, InMemoryGuardrailHandler, - get_guardrail_initializer_from_hooks, ) -from litellm.types.guardrails import Guardrail, GuardrailEventHooks, LitellmParams, LoggingOnlyScope, Mode +from litellm.types.guardrails import GuardrailEventHooks, Guardrail, LitellmParams, LoggingOnlyScope, Mode from litellm.types.utils import GenericGuardrailAPIInputs diff --git a/tests/unit/integrations/test_custom_guardrail.py b/tests/unit/integrations/test_custom_guardrail.py index 2fdb338f20e..d01b54aa953 100644 --- a/tests/unit/integrations/test_custom_guardrail.py +++ b/tests/unit/integrations/test_custom_guardrail.py @@ -546,10 +546,14 @@ class TestApplyGuardrailCheck: class ParentGuardrail(CustomGuardrail): """Parent that inherits apply_guardrail from CustomGuardrail""" + pass + # Child class that only inherits apply_guardrail (doesn't override) class ChildGuardrailWithoutOverride(ParentGuardrail): """Child that only inherits apply_guardrail""" + pass + # Child class that overrides apply_guardrail class ChildGuardrailWithOverride(ParentGuardrail): """Child that overrides apply_guardrail""" @@ -2536,7 +2540,7 @@ class TestLoggingOnlyApplyGuardrail: from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( ContentFilterGuardrail, ) - from litellm.types.guardrails import BlockedWord, ContentFilterAction + from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks guardrail: Final = ContentFilterGuardrail( guardrail_name="content-review", @@ -2628,7 +2632,7 @@ class TestLoggingOnlyApplyGuardrail: @pytest.mark.asyncio async def test_block_verdict_is_recorded_without_raising(self): - guardrail: Final = _ApplyOnlyObserver(block=True) + guardrail = _ApplyOnlyObserver(block=True) kwargs, response = _logged_call([{"role": "user", "content": "flagged content"}]) out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) @@ -2637,7 +2641,7 @@ class TestLoggingOnlyApplyGuardrail: ("request", ["flagged content"]), ("response", ["general kenobi"]), ] - entries: Final = out_kwargs["standard_logging_object"]["guardrail_information"] + entries = out_kwargs["standard_logging_object"]["guardrail_information"] assert [entry["guardrail_status"] for entry in entries] == ["guardrail_intervened", "guardrail_intervened"] @pytest.mark.asyncio @@ -2964,7 +2968,9 @@ async def test_native_lifecycle_guardrail_logging_only_scans_assembled_response( from litellm.types.utils import Choices, Message, ModelResponse guardrail = _NativeLifecycleLoggingGuardrail() - assembled = ModelResponse(choices=[Choices(message=Message(role="assistant", content="assembled stream text"))]) + assembled = ModelResponse( + choices=[Choices(message=Message(role="assistant", content="assembled stream text"))] + ) sentinel_result = object() kwargs = { "model": "gpt-5.4-mini", From 76c02ea7e910ed7315e3932b4e45a64d7cd55c7a Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 08:25:44 +0000 Subject: [PATCH 03/20] fix(guardrails): allow native lifecycle logging-only scope Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 1 - .../guardrails/test_guardrail_registry.py | 19 ++++++++++++++++++- .../integrations/test_custom_guardrail.py | 11 +++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index ffe9544007c..2407c0032a3 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -809,7 +809,6 @@ class CustomGuardrail(CustomLogger): def supports_logging_only_scope(self) -> bool: return ( self.uses_apply_guardrail_interface() - and not self.use_native_lifecycle_hooks and type(self).async_logging_hook is CustomGuardrail.async_logging_hook ) diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 6af7a7a06a4..45e652bff72 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -1,5 +1,5 @@ from collections.abc import Iterable -from typing import Final +from typing import ClassVar, Final from unittest.mock import AsyncMock, MagicMock import pytest @@ -956,6 +956,10 @@ class _LoggingOnlyScopeUnsupportedGuardrail(_LoggingOnlyScopeSupportedGuardrail) return kwargs, result +class _LoggingOnlyScopeNativeGuardrail(_LoggingOnlyScopeSupportedGuardrail): + use_native_lifecycle_hooks: ClassVar[bool] = True + + class TestLoggingOnlyScopeValidation: def _initialize( self, @@ -968,9 +972,13 @@ class TestLoggingOnlyScopeValidation: guardrail_type: Final = "logging_only_scope_test" def _initializer(litellm_params: LitellmParams, guardrail: Guardrail) -> CustomGuardrail: + supported_event_hooks: Final = ( + [GuardrailEventHooks.logging_only] if callback_type.use_native_lifecycle_hooks else None + ) return callback_type( guardrail_name=guardrail["guardrail_name"], event_hook=litellm_params.mode, + supported_event_hooks=supported_event_hooks, ) registry_module.guardrail_initializer_registry[guardrail_type] = _initializer @@ -1041,6 +1049,15 @@ class TestLoggingOnlyScopeValidation: assert callback.logging_only_scope == "both" + def test_output_scope_accepted_for_native_lifecycle_guardrail(self) -> None: + callback: Final = self._initialize( + mode="logging_only", + scope="output", + callback_type=_LoggingOnlyScopeNativeGuardrail, + ) + + assert callback.logging_only_scope == "output" + def test_invalid_scope_fails_litellm_params_validation(self) -> None: with pytest.raises(ValidationError): LitellmParams(guardrail="test", mode="logging_only", logging_only_scope="request") diff --git a/tests/unit/integrations/test_custom_guardrail.py b/tests/unit/integrations/test_custom_guardrail.py index d01b54aa953..052eb347b9c 100644 --- a/tests/unit/integrations/test_custom_guardrail.py +++ b/tests/unit/integrations/test_custom_guardrail.py @@ -2961,6 +2961,17 @@ class _NativeLifecycleLoggingGuardrail(CustomGuardrail): return inputs +@pytest.mark.asyncio +async def test_native_lifecycle_guardrail_logging_only_scope_scans_only_input(): + guardrail: Final = _NativeLifecycleLoggingGuardrail() + guardrail.logging_only_scope = "input" + kwargs, response = _logged_call([{"role": "user", "content": "native lifecycle input"}]) + + await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [("request", ["native lifecycle input"])] + + @pytest.mark.asyncio async def test_native_lifecycle_guardrail_logging_only_scans_assembled_response(): """A use_native_lifecycle_hooks guardrail accepts mode logging_only and its From d392424670f85e0909785beaf44714d1795e9fc7 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 08:29:13 +0000 Subject: [PATCH 04/20] chore(ui): regenerate api types for logging_only_scope Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_lazy_openapi_snapshot.json | 34 +++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 ++++++ 2 files changed, 44 insertions(+) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 75dce43c84a..98bdb22dc12 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -10087,6 +10087,23 @@ "description": "Google Cloud location/region (e.g., us-central1)", "title": "Location" }, + "logging_only_scope": { + "anyOf": [ + { + "enum": [ + "input", + "output", + "both" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "which direction a logging_only scan observes: 'input' (request), 'output' (response), or 'both' (default). Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking.", + "title": "Logging Only Scope" + }, "mask_request_content": { "anyOf": [ { @@ -12325,6 +12342,23 @@ "description": "Google Cloud location/region (e.g., us-central1)", "title": "Location" }, + "logging_only_scope": { + "anyOf": [ + { + "enum": [ + "input", + "output", + "both" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "which direction a logging_only scan observes: 'input' (request), 'output' (response), or 'both' (default). Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking.", + "title": "Logging Only Scope" + }, "mask": { "anyOf": [ { diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 93f4718a586..322cb7c1e0c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -25437,6 +25437,11 @@ export interface components { * @description Google Cloud location/region (e.g., us-central1) */ location?: string | null; + /** + * Logging Only Scope + * @description which direction a logging_only scan observes: 'input' (request), 'output' (response), or 'both' (default). Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking. + */ + logging_only_scope?: ("input" | "output" | "both") | null; /** * Mask Request Content * @description Will mask request content if guardrail makes any changes @@ -34431,6 +34436,11 @@ export interface components { * @description Google Cloud location/region (e.g., us-central1) */ location?: string | null; + /** + * Logging Only Scope + * @description which direction a logging_only scan observes: 'input' (request), 'output' (response), or 'both' (default). Only applies to mode logging_only; pre_call/post_call on the same guardrail keep blocking. + */ + logging_only_scope?: ("input" | "output" | "both") | null; /** * Mask * @description Enable content masking using Lasso classifix API From 419f362294a9806da79c17cccc01e6fb1638a2b8 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 09:27:54 +0000 Subject: [PATCH 05/20] fix(guardrails): remove callbacks when scope validation fails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 2 +- .../proxy/guardrails/guardrail_registry.py | 9 ++- .../observability/test_guardrail_effects.py | 59 +++++++++++++++++++ .../guardrails/test_guardrail_registry.py | 11 +++- 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 2407c0032a3..b1291d663bb 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -943,7 +943,7 @@ class CustomGuardrail(CustomLogger): result: object, call_type: str, ) -> tuple[dict, object]: # mutable-ok: CustomLogger.async_logging_hook contract - """logging_only: run apply_guardrail on copies of the logged request/response and record the verdict.""" + """logging_only: scan copies of the logged request and/or response according to logging_only_scope.""" from litellm.llms import get_guardrail_translation_mapping if not self.uses_apply_guardrail_interface(): diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 6527f6be50b..0258b291d21 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -558,8 +558,13 @@ class InMemoryGuardrailHandler: config_file_path=config_file_path, llm_router=llm_router, ) - for custom_guardrail_callback in created_callbacks: - _configure_callback_scoping(custom_guardrail_callback, guardrail["guardrail_name"], litellm_params) + try: + for custom_guardrail_callback in created_callbacks: + _configure_callback_scoping(custom_guardrail_callback, guardrail["guardrail_name"], litellm_params) + except Exception: + for custom_guardrail_callback in created_callbacks: + litellm.logging_callback_manager.remove_callback_from_all_lists(custom_guardrail_callback) + raise parsed_guardrail: Final = Guardrail( guardrail_id=guardrail.get("guardrail_id"), diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index 4e9691bd4e9..ad487b1f718 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -1391,3 +1391,62 @@ def test_logging_only_scope_observes_only_the_configured_direction_without_block return_last_on_timeout=True, ) assert detail["requestsEvaluated"] == len(scanned_directions), detail + + +def test_logging_only_scope_without_logging_only_mode_is_skipped_at_load_and_never_blocks( + gateway: Gateway, tmp_path: Path +) -> None: + identity: Final = "guardrail" + uuid.uuid4().hex + prompt: Final = "synthetic invalid-scope prompt " + identity + reply: Final = "synthetic invalid-scope reply " + identity + + def guardrail(request: Request) -> Reply: + assert request.target == "/beta/litellm_basic_guardrail_api" + return Reply(body=json.dumps({"action": "BLOCKED", "blocked_reason": "synthetic policy denial"}).encode()) + + def provider(request: Request) -> Reply: + assert request.target == "/v1/chat/completions" + assert json.loads(request.body)["messages"] == [{"role": "user", "content": prompt}] + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": reply}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14}, + } + ).encode() + ) + + with wire_server(guardrail) as policy, wire_server(provider) as upstream: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["guardrails"] = [ + { + "guardrail_name": identity, + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "logging_only_scope": "input", + "default_on": True, + "api_base": policy.url, + "api_key": "synthetic-guardrail-key", + }, + } + ] + path: Final = tmp_path / "invalid-scope-pre-call.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate, candidate.scenario() as scenario: + model: Final = scenario.model(api_base=upstream.url + "/v1") + response: Final = candidate.request( + "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": prompt}]} + ) + assert response.status_code == 200, response.text + assert response.json()["choices"][0]["message"]["content"] == reply, response.text + assert len(policy.drain()) == 0 + assert len(upstream.drain()) == 1 + guardrails: Final = candidate.get("/v2/guardrails/list")["guardrails"] + assert all(object_value(row)["guardrail_name"] != identity for row in guardrails), guardrails diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index 45e652bff72..ddfdabf689d 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -967,19 +967,24 @@ class TestLoggingOnlyScopeValidation: scope: LoggingOnlyScope | None, callback_type: type[CustomGuardrail] = _LoggingOnlyScopeSupportedGuardrail, ) -> CustomGuardrail: + import litellm from litellm.proxy.guardrails import guardrail_registry as registry_module guardrail_type: Final = "logging_only_scope_test" + created_callbacks: Final[list[CustomGuardrail]] = [] def _initializer(litellm_params: LitellmParams, guardrail: Guardrail) -> CustomGuardrail: supported_event_hooks: Final = ( [GuardrailEventHooks.logging_only] if callback_type.use_native_lifecycle_hooks else None ) - return callback_type( + callback: Final = callback_type( guardrail_name=guardrail["guardrail_name"], event_hook=litellm_params.mode, supported_event_hooks=supported_event_hooks, ) + litellm.logging_callback_manager.add_litellm_callback(callback) + created_callbacks.append(callback) + return callback registry_module.guardrail_initializer_registry[guardrail_type] = _initializer lists: Final = _all_callback_lists() @@ -1000,6 +1005,10 @@ class TestLoggingOnlyScopeValidation: callback: Final = handler.guardrail_id_to_custom_guardrail[result["guardrail_id"]] assert callback is not None return callback + except ValueError: + callback: Final = created_callbacks[0] + assert all(callback not in callback_list for callback_list in lists) + raise finally: for callback_list, snapshot in zip(lists, snapshots): callback_list[:] = snapshot From 1d0db0fbc7f7c5dc12ed545a8d442fccef1fd7e0 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 09:39:13 +0000 Subject: [PATCH 06/20] fix(guardrails): keep output-only scans when request copy fails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 20 ++++++++++++++++--- .../integrations/test_custom_guardrail.py | 17 ++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index b1291d663bb..42e103f942b 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -990,6 +990,21 @@ class CustomGuardrail(CustomLogger): "standard_logging_object": {**standard_logging_object, "guardrail_information": [*existing, *entries]}, }, result + def _copy_scratch_request_fields( + self, + kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract + ) -> tuple[object, object]: + optional_params: Final = kwargs.get("optional_params") or {} + try: + return ( + copy.deepcopy(kwargs.get("messages") or kwargs.get("input")), + copy.deepcopy(optional_params.get("tools")), + ) + except Exception: + if self.logging_only_scope == "output": + return None, None + raise + async def _scan_logged_call( self, kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract @@ -998,13 +1013,12 @@ class CustomGuardrail(CustomLogger): output_translation: "BaseTranslation", scratch_metadata: dict, # mutable-ok: apply_guardrail records its verdict into request metadata ) -> None: - optional_params: Final = kwargs.get("optional_params") or {} - scratch_input: Final = copy.deepcopy(kwargs.get("messages") or kwargs.get("input")) + scratch_input, scratch_tools = self._copy_scratch_request_fields(kwargs) scratch_request: Final = { "model": kwargs.get("model"), "messages": scratch_input, "input": scratch_input, - "tools": copy.deepcopy(optional_params.get("tools")), + "tools": scratch_tools, "litellm_call_id": kwargs.get("litellm_call_id"), "metadata": scratch_metadata, } diff --git a/tests/unit/integrations/test_custom_guardrail.py b/tests/unit/integrations/test_custom_guardrail.py index 052eb347b9c..c31379e7f1d 100644 --- a/tests/unit/integrations/test_custom_guardrail.py +++ b/tests/unit/integrations/test_custom_guardrail.py @@ -2630,6 +2630,23 @@ class TestLoggingOnlyApplyGuardrail: assert out_kwargs is kwargs assert out_response is response + @pytest.mark.asyncio + async def test_output_scope_scans_response_when_request_copy_fails(self): + import threading + + guardrail: Final = _ApplyOnlyObserver() + guardrail.logging_only_scope = "output" + call: Final = _logged_call([{"role": "user", "content": "hello there", "lock": threading.Lock()}]) + kwargs: Final = call[0] + response: Final = call[1] + + out_kwargs, _ = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) + + assert guardrail.calls == [("response", ["general kenobi"])] + entries: Final = out_kwargs["standard_logging_object"]["guardrail_information"] + assert [entry["guardrail_name"] for entry in entries] == ["apply-only-observer"] + assert [entry["guardrail_status"] for entry in entries] == ["success"] + @pytest.mark.asyncio async def test_block_verdict_is_recorded_without_raising(self): guardrail = _ApplyOnlyObserver(block=True) From fa5f7904a9ea4ece23f8b549cea104cfc9a1f382 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 09:48:51 +0000 Subject: [PATCH 07/20] feat(ui): configure logging_only_scope on guardrails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../_components/GuardrailFormField.tsx | 48 +++++++++++++++++ .../_components/GuardrailModeDisplay.tsx | 47 +++++++++++++++++ ...d_guardrail_form.characterization.test.tsx | 42 +++++++++++++++ .../_components/add_guardrail_form.tsx | 20 +++++++- .../custom_code/CustomCodeModal.tsx | 2 + .../guardrail_info.characterization.test.tsx | 45 ++++++++++++++++ .../guardrails/_components/guardrail_info.tsx | 29 +++++------ .../guardrail_info_helpers.test.tsx | 51 +++++++++++++++++++ .../_components/guardrail_info_helpers.tsx | 36 +++++++++++++ 9 files changed, 302 insertions(+), 18 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailModeDisplay.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx index 0afd9aab0cf..53a624d4422 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx @@ -6,6 +6,7 @@ import { useController, type Control, type ControllerRenderProps, type RegisterO import { Field, FieldDescription, FieldError, FieldLabel } from "@/components/ui/field"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { modeIncludesLoggingOnly, type LoggingOnlyScopeChoice } from "./guardrail_info_helpers"; export interface GuardrailCriterion { name: string; @@ -15,6 +16,7 @@ export interface GuardrailCriterion { export interface GuardrailFormValues extends Record { criteria?: GuardrailCriterion[]; + logging_only_scope_choice?: LoggingOnlyScopeChoice; } export type GuardrailFormControl = Control; export type GuardrailFieldRules = Pick, "validate">; @@ -123,3 +125,49 @@ export const SkipMessageSelect: React.FC<{ control: GuardrailFieldControlProps } ); }; + +const LOGGING_ONLY_SCOPE_ITEMS: Array<{ label: string; value: LoggingOnlyScopeChoice }> = [ + { label: "Default (request and response)", value: "default" }, + { label: "Input only (request)", value: "input" }, + { label: "Output only (response)", value: "output" }, + { label: "Both (request and response)", value: "both" }, +]; + +export const LoggingOnlyScopeSelect: React.FC<{ control: GuardrailFieldControlProps }> = ({ control }) => { + const { id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy } = control; + + return ( + + ); +}; + +export const LoggingOnlyScopeField: React.FC<{ control: GuardrailFormControl; mode: unknown }> = ({ + control, + mode, +}) => { + if (!modeIncludesLoggingOnly(mode)) return null; + + return ( + + {(fieldControl) => } + + ); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailModeDisplay.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailModeDisplay.tsx new file mode 100644 index 00000000000..fe45e183e82 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailModeDisplay.tsx @@ -0,0 +1,47 @@ +import React from "react"; +import { Badge } from "@/components/ui/badge"; +import { Card } from "@/components/ui/card"; +import { + formatGuardrailMode, + formatLoggingOnlyScope, + modeIncludesLoggingOnly, +} from "./guardrail_info_helpers"; + +type GuardrailModeParams = { + mode?: unknown; + default_on?: boolean; + logging_only_scope?: string | null; +}; + +export const GuardrailModeCard: React.FC<{ litellmParams: GuardrailModeParams }> = ({ litellmParams }) => ( + +

Mode

+
+

{formatGuardrailMode(litellmParams.mode) || "-"}

+ + {litellmParams.default_on ? "Default On" : "Default Off"} + +
+ {modeIncludesLoggingOnly(litellmParams.mode) && ( +
+

Logging only scope

+

{formatLoggingOnlyScope(litellmParams.logging_only_scope)}

+
+ )} +
+); + +export const GuardrailModeRows: React.FC<{ litellmParams: GuardrailModeParams }> = ({ litellmParams }) => ( + <> +
+

Mode

+
{formatGuardrailMode(litellmParams.mode) || "-"}
+
+ {modeIncludesLoggingOnly(litellmParams.mode) && ( +
+

Logging only scope

+
{formatLoggingOnlyScope(litellmParams.logging_only_scope)}
+
+ )} + +); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx index aadc9ec0213..89a00b08d6b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx @@ -105,6 +105,48 @@ describe("AddGuardrailForm create payload characterization", () => { expect(payload()).toMatchObject({ litellm_params: { mode: ["pre_call", "post_call"] } }); }); + it("sends the selected output logging-only scope", async () => { + vi.mocked(networking.getGuardrailUISettings).mockResolvedValue({ + ...uiSettings, + supported_modes: ["pre_call", "logging_only"], + }); + const user = userEvent.setup({ delay: null }); + renderForm(); + + await user.type(await screen.findByLabelText("Guardrail Name"), "my-bedrock"); + await pickProvider(user, "Bedrock Guardrail"); + await user.click(screen.getByLabelText("Mode")); + await user.click((await screen.findAllByText("logging_only")).at(-1) as HTMLElement); + await chooseSelectOption( + user, + await screen.findByLabelText("Logging only scope"), + "Output only (response)", + ); + + await user.type(await screen.findByPlaceholderText("The guardrail id on Bedrock"), "gr-123"); + await user.click(screen.getByRole("button", { name: "Next" })); + await user.click(await screen.findByRole("button", { name: "Create Guardrail" })); + + await waitFor(() => expect(networking.createGuardrailCall).toHaveBeenCalledTimes(1)); + expect(payload()?.litellm_params.logging_only_scope).toBe("output"); + }); + + it("hides logging-only scope and omits it from a pre-call payload", async () => { + const user = userEvent.setup({ delay: null }); + renderForm(); + + await user.type(await screen.findByLabelText("Guardrail Name"), "my-bedrock"); + await pickProvider(user, "Bedrock Guardrail"); + expect(screen.queryByLabelText("Logging only scope")).not.toBeInTheDocument(); + + await user.type(await screen.findByPlaceholderText("The guardrail id on Bedrock"), "gr-123"); + await user.click(screen.getByRole("button", { name: "Next" })); + await user.click(await screen.findByRole("button", { name: "Create Guardrail" })); + + await waitFor(() => expect(networking.createGuardrailCall).toHaveBeenCalledTimes(1)); + expect(payload()?.litellm_params).not.toHaveProperty("logging_only_scope"); + }); + it("blocks Next when the user deselects every mode", async () => { const user = userEvent.setup({ delay: null }); renderForm(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx index 29df7c8bf3d..689dbbfefa2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx @@ -1,5 +1,5 @@ import React, { useEffect, useMemo, useState } from "react"; -import { useForm, type UseFormReturn } from "react-hook-form"; +import { useForm, useWatch, type UseFormReturn } from "react-hook-form"; import { toast } from "@/lib/toast"; import { createGuardrailCall, @@ -10,18 +10,22 @@ import { import ContentFilterConfiguration from "./content_filter/ContentFilterConfiguration"; import { type CompetitorIntentConfig } from "./content_filter/CompetitorIntentConfiguration"; import { + choiceToLoggingOnlyScope, choiceToSkipSystemForCreate, choiceToSkipToolForCreate, getGuardrailLogo, getGuardrailProviders, getSupportedModesForProvider, guardrail_provider_map, + modeIncludesLoggingOnly, populateGuardrailProviderMap, populateGuardrailProviders, shouldRenderContentFilterConfigSettings, shouldRenderLLMJudgeFields, shouldRenderPIIConfigSettings, toModeArray, + type LoggingOnlyScope, + type LoggingOnlyScopeChoice, } from "./guardrail_info_helpers"; import { Logo } from "@/components/molecules/logo/Logo"; import { MultiSelect } from "@/components/shared/MultiSelect"; @@ -49,6 +53,7 @@ import { requiredRule, type GuardrailCriterion, type GuardrailFormValues, + LoggingOnlyScopeField, SkipMessageSelect, } from "./GuardrailFormField"; import GuardrailOptionalParams from "./guardrail_optional_params"; @@ -160,6 +165,7 @@ type SkipMessageChoice = "inherit" | "yes" | "no"; const INITIAL_VALUES: GuardrailFormValues = { mode: "pre_call", default_on: false, + logging_only_scope_choice: "default", skip_system_message_choice: "inherit", skip_tool_message_choice: "inherit", }; @@ -199,6 +205,7 @@ interface ProviderParamsResponse { const AddGuardrailForm: React.FC = ({ visible, onClose, accessToken, onSuccess, preset }) => { const form = useForm({ defaultValues: INITIAL_VALUES }); + const watchedMode = useWatch({ control: form.control, name: "mode" }); const [loading, setLoading] = useState(false); const [selectedProvider, setSelectedProvider] = useState(null); const [guardrailSettings, setGuardrailSettings] = useState(null); @@ -277,6 +284,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a guardrail_name: preset.guardrailNameSuggestion, mode: preset.mode, default_on: preset.defaultOn, + logging_only_scope_choice: "default", skip_system_message_choice: "inherit", skip_tool_message_choice: "inherit", }; @@ -439,6 +447,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a guardrail_name: string; litellm_params: { guardrail: string; + logging_only_scope?: LoggingOnlyScope | null; [key: string]: unknown; // Allow dynamic properties }; guardrail_info: Record; @@ -462,6 +471,13 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a guardrailData.litellm_params.skip_tool_message_in_guardrail = skipToolForCreate; } + const loggingOnlyScope = choiceToLoggingOnlyScope( + values.logging_only_scope_choice as LoggingOnlyScopeChoice | undefined, + ); + if (modeIncludesLoggingOnly(values.mode) && loggingOnlyScope !== null) { + guardrailData.litellm_params.logging_only_scope = loggingOnlyScope; + } + // For Presidio PII, add the entity and action configurations if (providerKey === "PresidioPII" && selectedEntities.length > 0) { const piiEntitiesConfig: { [key: string]: string } = {}; @@ -796,6 +812,8 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a {(fieldControl) => } + + {/* Use the GuardrailProviderFields component to render provider-specific fields */} {showProviderFields && ( { expect(lastPayload()).toEqual({ litellm_params: { skip_system_message_in_guardrail: true } }); }); + it("shows and updates the logging-only scope", async () => { + vi.mocked(networking.getGuardrailInfo).mockResolvedValue( + guardrail({ + guardrailIdentifier: "gr-abc", + api_key: "sk-old", + mode: "logging_only", + logging_only_scope: "input", + }), + ); + const user = userEvent.setup({ delay: null }); + renderView(); + + expect(await screen.findAllByText("Input only (request)")).toHaveLength(2); + await openEditor(user); + await chooseSelectOption( + user, + screen.getByLabelText("Logging only scope"), + "Output only (response)", + ); + await saveChanges(user); + + await waitFor(() => expect(networking.updateGuardrailCall).toHaveBeenCalledTimes(1)); + expect(lastPayload()).toEqual({ litellm_params: { logging_only_scope: "output" } }); + }); + + it("clears the logging-only scope when the edit choice returns to default", async () => { + vi.mocked(networking.getGuardrailInfo).mockResolvedValue( + guardrail({ + guardrailIdentifier: "gr-abc", + api_key: "sk-old", + mode: "logging_only", + logging_only_scope: "input", + }), + ); + const user = userEvent.setup({ delay: null }); + renderView(); + await openEditor(user); + + await chooseSelectOption(user, screen.getByLabelText("Logging only scope"), "Default (request and response)"); + await saveChanges(user); + + await waitFor(() => expect(networking.updateGuardrailCall).toHaveBeenCalledTimes(1)); + expect(lastPayload()).toEqual({ litellm_params: { logging_only_scope: null } }); + }); + it("parses the guardrail information textarea into an object", async () => { const user = userEvent.setup({ delay: null }); renderView(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx index c9162d99934..e75e55244de 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx @@ -28,14 +28,17 @@ import { readRecord, requiredRule, type GuardrailFormValues, + LoggingOnlyScopeField, SkipMessageSelect, } from "./GuardrailFormField"; import ContentFilterManager, { formatContentFilterDataForAPI } from "./content_filter/ContentFilterManager"; import CustomCodeModal, { EditGuardrailData } from "./custom_code/CustomCodeModal"; +import { GuardrailModeCard, GuardrailModeRows } from "./GuardrailModeDisplay"; import { - formatGuardrailMode, + getLoggingOnlyScopeUpdate, getGuardrailLogoAndName, guardrail_provider_map, + loggingOnlyScopeToChoice, skipSystemMessageToChoice, skipToolMessageToChoice, type SkipSystemMessageChoice, @@ -219,6 +222,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, if (!guardrailData) return; form.setValue("guardrail_name", guardrailData.guardrail_name); form.setValue("default_on", guardrailData.litellm_params?.default_on); + form.setValue("logging_only_scope_choice", loggingOnlyScopeToChoice(guardrailData.litellm_params?.logging_only_scope)); form.setValue( "skip_system_message_choice", skipSystemMessageToChoice(guardrailData.litellm_params?.skip_system_message_in_guardrail), @@ -282,7 +286,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, // Prepare update data object - only include changed fields const updateData: any = { - litellm_params: {}, + litellm_params: getLoggingOnlyScopeUpdate(guardrailData.litellm_params, values.logging_only_scope_choice), }; // Only include guardrail_name if it has changed @@ -556,17 +560,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, - -

Mode

-
-

- {formatGuardrailMode(guardrailData.litellm_params?.mode) || "-"} -

- - {guardrailData.litellm_params?.default_on ? "Default On" : "Default Off"} - -
-
+

Created At

@@ -745,6 +739,10 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, > {(fieldControl) => } + {guardrailData.litellm_params?.guardrail === "presidio" && ( <> PII Protection @@ -856,10 +854,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose,

Provider

{displayName}
-
-

Mode

-
{formatGuardrailMode(guardrailData.litellm_params?.mode) || "-"}
-
+

Default On

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx index c5e07fe9624..95c66e40d6b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx @@ -15,6 +15,11 @@ import { skipToolMessageToChoice, choiceToSkipToolForCreate, formatGuardrailMode, + loggingOnlyScopeToChoice, + choiceToLoggingOnlyScope, + getLoggingOnlyScopeUpdate, + formatLoggingOnlyScope, + modeIncludesLoggingOnly, } from "./guardrail_info_helpers"; describe("guardrail_info_helpers", () => { @@ -239,6 +244,52 @@ describe("guardrail_info_helpers", () => { }); }); + describe("logging-only scope helpers", () => { + it("maps API scope values to choices and back", () => { + expect(loggingOnlyScopeToChoice("input")).toBe("input"); + expect(loggingOnlyScopeToChoice("output")).toBe("output"); + expect(loggingOnlyScopeToChoice("both")).toBe("both"); + expect(loggingOnlyScopeToChoice(undefined)).toBe("default"); + expect(loggingOnlyScopeToChoice(null)).toBe("default"); + expect(loggingOnlyScopeToChoice("invalid")).toBe("default"); + + expect(choiceToLoggingOnlyScope("default")).toBeNull(); + expect(choiceToLoggingOnlyScope(undefined)).toBeNull(); + expect(choiceToLoggingOnlyScope("input")).toBe("input"); + expect(choiceToLoggingOnlyScope("output")).toBe("output"); + expect(choiceToLoggingOnlyScope("both")).toBe("both"); + + expect(getLoggingOnlyScopeUpdate({ logging_only_scope: "input" }, "input")).toEqual({}); + expect(getLoggingOnlyScopeUpdate({ logging_only_scope: "input" }, "output")).toEqual({ + logging_only_scope: "output", + }); + expect(getLoggingOnlyScopeUpdate({ logging_only_scope: "input" }, "default")).toEqual({ + logging_only_scope: null, + }); + }); + + it("formats every scope and falls back to default for missing or unknown values", () => { + expect(formatLoggingOnlyScope("input")).toBe("Input only (request)"); + expect(formatLoggingOnlyScope("output")).toBe("Output only (response)"); + expect(formatLoggingOnlyScope("both")).toBe("Both (request and response)"); + expect(formatLoggingOnlyScope(undefined)).toBe("Default (request and response)"); + expect(formatLoggingOnlyScope(null)).toBe("Default (request and response)"); + expect(formatLoggingOnlyScope("invalid")).toBe("Default (request and response)"); + }); + + it("detects logging_only in string, array, and tagged mode values", () => { + expect(modeIncludesLoggingOnly("logging_only")).toBe(true); + expect(modeIncludesLoggingOnly(["pre_call", "logging_only"])).toBe(true); + expect( + modeIncludesLoggingOnly({ + tags: { "Service-Type: internal-service": "logging_only" }, + default: "pre_call", + }), + ).toBe(true); + expect(modeIncludesLoggingOnly("pre_call")).toBe(false); + }); + }); + describe("skipSystemMessageToChoice / choiceToSkipSystemForCreate", () => { it("maps API values to form choices and back for create", () => { expect(skipSystemMessageToChoice(undefined)).toBe("inherit"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx index 476bcd3a8ae..95d463a55a9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx @@ -114,6 +114,42 @@ export const toModeArray = (raw: unknown): string[] => { return []; }; +export type LoggingOnlyScope = "input" | "output" | "both"; +export type LoggingOnlyScopeChoice = "default" | LoggingOnlyScope; + +export const loggingOnlyScopeToChoice = (v: string | null | undefined): LoggingOnlyScopeChoice => + v === "input" || v === "output" || v === "both" ? v : "default"; + +export const choiceToLoggingOnlyScope = (choice: LoggingOnlyScopeChoice | undefined): LoggingOnlyScope | null => + choice === "input" || choice === "output" || choice === "both" ? choice : null; + +export const getLoggingOnlyScopeUpdate = ( + litellmParams: { logging_only_scope?: string | null } | null | undefined, + choice: LoggingOnlyScopeChoice | undefined, +): { logging_only_scope?: LoggingOnlyScope | null } => { + if (choice === undefined || choice === loggingOnlyScopeToChoice(litellmParams?.logging_only_scope)) return {}; + return { logging_only_scope: choiceToLoggingOnlyScope(choice) }; +}; + +export const formatLoggingOnlyScope = (v: string | null | undefined): string => { + if (v === "input") return "Input only (request)"; + if (v === "output") return "Output only (response)"; + if (v === "both") return "Both (request and response)"; + return "Default (request and response)"; +}; + +export const modeIncludesLoggingOnly = (raw: unknown): boolean => { + if (toModeArray(raw).includes("logging_only")) return true; + if (raw === null || typeof raw !== "object") return false; + + const { tags, default: fallback } = raw as { tags?: Record; default?: unknown }; + const taggedModes = + tags && typeof tags === "object" + ? Object.values(tags).some((mode) => toModeArray(mode).includes("logging_only")) + : false; + return toModeArray(fallback).includes("logging_only") || taggedModes; +}; + export const formatGuardrailMode = (raw: unknown): string => { const flat: string[] = toModeArray(raw); if (flat.length > 0) return flat.join(", "); From e755b9a994366a55bcdaa25356822e28353dc288 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 10:00:50 +0000 Subject: [PATCH 08/20] fix(guardrails): keep guardrails enforcing when logging_only_scope is invalid at load Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/guardrails/guardrail_endpoints.py | 12 +++- .../proxy/guardrails/guardrail_registry.py | 68 ++++++++++++++++--- .../observability/test_guardrail_effects.py | 22 +++--- .../guardrails/test_guardrail_endpoints.py | 38 ++++++++++- .../guardrails/test_guardrail_registry.py | 31 +++++++-- 5 files changed, 145 insertions(+), 26 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 6053ab26726..d52996cd68d 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -403,7 +403,11 @@ async def create_guardrail( guardrail_id: Final = result.get("guardrail_id", "Unknown") try: - IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail(guardrail=cast(Guardrail, result), source="db") + IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail( + guardrail=cast(Guardrail, result), + source="db", + reject_invalid_logging_only_scope=True, + ) verbose_proxy_logger.info( "Immediate sync: Successfully initialized guardrail '%s' (ID: %s)", guardrail_name, guardrail_id ) @@ -526,7 +530,10 @@ async def update_guardrail( guardrail_name: Final = result.get("guardrail_name", "Unknown") try: - IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db(guardrail=cast(Guardrail, result)) + IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db( + guardrail=cast(Guardrail, result), + reject_invalid_logging_only_scope=True, + ) verbose_proxy_logger.info( "Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id ) @@ -1246,6 +1253,7 @@ async def patch_guardrail( try: IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db( guardrail=guardrail, + reject_invalid_logging_only_scope=True, ) verbose_proxy_logger.info( "Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 0258b291d21..5f071083683 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -437,23 +437,46 @@ def _as_callback_tuple( return (initialized,) -def _configure_callback_scoping( +def _logging_only_scope_error( custom_guardrail_callback: CustomGuardrail, guardrail_name: str, litellm_params: LitellmParams -) -> None: +) -> str | None: logging_only_scope: Final = litellm_params.logging_only_scope - custom_guardrail_callback.logging_only_scope = logging_only_scope if logging_only_scope is not None and GuardrailEventHooks.logging_only.value not in _configured_event_hooks( litellm_params.mode ): - raise ValueError( + return ( f"Guardrail {guardrail_name}: logging_only_scope is set, but mode does not include logging_only, " "so it would never apply. Add logging_only to mode or remove logging_only_scope." ) if logging_only_scope in ("input", "output") and not custom_guardrail_callback.supports_logging_only_scope(): - raise ValueError( + return ( f"Guardrail {guardrail_name}: logging_only_scope={logging_only_scope!r} is not supported by this " "guardrail, whose logging_only hook scans on its own. Remove logging_only_scope." ) + return None + + +def _configure_callback_scoping( + custom_guardrail_callback: CustomGuardrail, + guardrail_name: str, + litellm_params: LitellmParams, + *, + reject_invalid_logging_only_scope: bool = False, +) -> None: + logging_only_scope: Final = litellm_params.logging_only_scope + logging_only_scope_error: Final = _logging_only_scope_error( + custom_guardrail_callback, guardrail_name, litellm_params + ) + if logging_only_scope_error is not None: + if reject_invalid_logging_only_scope: + raise ValueError(logging_only_scope_error) + verbose_proxy_logger.error( + "%s Ignoring logging_only_scope; the guardrail keeps its configured mode.", + logging_only_scope_error, + ) + custom_guardrail_callback.logging_only_scope = None + else: + custom_guardrail_callback.logging_only_scope = logging_only_scope for scoping_param in ( "skip_system_message_in_guardrail", "skip_tool_message_in_guardrail", @@ -512,6 +535,8 @@ class InMemoryGuardrailHandler: config_file_path: str | None = None, llm_router: Optional["Router"] = None, source: Literal["db", "config"] = "config", + *, + reject_invalid_logging_only_scope: bool = False, ) -> Guardrail | None: """ Initialize a guardrail from a dictionary and add it to the litellm callback manager @@ -560,7 +585,12 @@ class InMemoryGuardrailHandler: ) try: for custom_guardrail_callback in created_callbacks: - _configure_callback_scoping(custom_guardrail_callback, guardrail["guardrail_name"], litellm_params) + _configure_callback_scoping( + custom_guardrail_callback, + guardrail["guardrail_name"], + litellm_params, + reject_invalid_logging_only_scope=reject_invalid_logging_only_scope, + ) except Exception: for custom_guardrail_callback in created_callbacks: litellm.logging_callback_manager.remove_callback_from_all_lists(custom_guardrail_callback) @@ -672,6 +702,8 @@ class InMemoryGuardrailHandler: guardrail_id: str, guardrail: Guardrail, source: Literal["db", "config"] = "db", + *, + reject_invalid_logging_only_scope: bool = False, ) -> None: """ Update a guardrail in memory: a changed name or litellm_params rebuilds the @@ -680,7 +712,11 @@ class InMemoryGuardrailHandler: """ updated_guardrail: Final = cast(Guardrail, {**guardrail, "guardrail_id": guardrail_id}) if self._has_guardrail_params_changed(guardrail_id, updated_guardrail): - self.reinitialize_guardrail(guardrail=updated_guardrail, source=source) + self.reinitialize_guardrail( + guardrail=updated_guardrail, + source=source, + reject_invalid_logging_only_scope=reject_invalid_logging_only_scope, + ) return self.IN_MEMORY_GUARDRAILS[guardrail_id] = updated_guardrail self._sources[guardrail_id] = source @@ -833,6 +869,8 @@ class InMemoryGuardrailHandler: guardrail: Guardrail, config_file_path: str | None = None, source: Literal["db", "config"] = "config", + *, + reject_invalid_logging_only_scope: bool = False, ) -> Guardrail | None: """ Force re-initialization of a guardrail even if it exists in memory. @@ -862,7 +900,12 @@ class InMemoryGuardrailHandler: # instance instead of leaving the guardrail silently removed: a guardrail # that was enforcing must never fail open because an update was bad. try: - return self.initialize_guardrail(guardrail=guardrail, config_file_path=config_file_path, source=source) + return self.initialize_guardrail( + guardrail=guardrail, + config_file_path=config_file_path, + source=source, + reject_invalid_logging_only_scope=reject_invalid_logging_only_scope, + ) except Exception as init_error: if previous_guardrail is not None: verbose_proxy_logger.exception( @@ -877,7 +920,13 @@ class InMemoryGuardrailHandler: verbose_proxy_logger.exception("Restoring previous guardrail %s also failed", guardrail_id) raise ValueError(f"Guardrail initialization failed: {init_error}") from init_error - def sync_guardrail_from_db(self, guardrail: Guardrail, config_file_path: str | None = None) -> Guardrail | None: + def sync_guardrail_from_db( + self, + guardrail: Guardrail, + config_file_path: str | None = None, + *, + reject_invalid_logging_only_scope: bool = False, + ) -> Guardrail | None: """ Sync a guardrail from DB - initializes if new, re-initializes if changed. This is the method to call during DB polling. @@ -896,6 +945,7 @@ class InMemoryGuardrailHandler: guardrail=guardrail, config_file_path=config_file_path, source="db", + reject_invalid_logging_only_scope=reject_invalid_logging_only_scope, ) # Params unchanged but the entry is still DB-backed; make sure the diff --git a/tests/integration/observability/test_guardrail_effects.py b/tests/integration/observability/test_guardrail_effects.py index ad487b1f718..cf182188e86 100644 --- a/tests/integration/observability/test_guardrail_effects.py +++ b/tests/integration/observability/test_guardrail_effects.py @@ -1393,12 +1393,11 @@ def test_logging_only_scope_observes_only_the_configured_direction_without_block assert detail["requestsEvaluated"] == len(scanned_directions), detail -def test_logging_only_scope_without_logging_only_mode_is_skipped_at_load_and_never_blocks( +def test_logging_only_scope_without_logging_only_mode_is_ignored_at_load_and_keeps_blocking( gateway: Gateway, tmp_path: Path ) -> None: identity: Final = "guardrail" + uuid.uuid4().hex - prompt: Final = "synthetic invalid-scope prompt " + identity - reply: Final = "synthetic invalid-scope reply " + identity + prompt: Final = "synthetic invalid-scope prompt pineapple " + identity def guardrail(request: Request) -> Reply: assert request.target == "/beta/litellm_basic_guardrail_api" @@ -1415,7 +1414,11 @@ def test_logging_only_scope_without_logging_only_mode_is_skipped_at_load_and_nev "created": 1, "model": "gpt-4o-mini", "choices": [ - {"index": 0, "message": {"role": "assistant", "content": reply}, "finish_reason": "stop"} + { + "index": 0, + "message": {"role": "assistant", "content": "unchanged provider reply"}, + "finish_reason": "stop", + } ], "usage": {"prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14}, } @@ -1432,6 +1435,7 @@ def test_logging_only_scope_without_logging_only_mode_is_skipped_at_load_and_nev "mode": "pre_call", "logging_only_scope": "input", "default_on": True, + "blocked_words": [{"keyword": "pineapple", "action": "BLOCK"}], "api_base": policy.url, "api_key": "synthetic-guardrail-key", }, @@ -1444,9 +1448,9 @@ def test_logging_only_scope_without_logging_only_mode_is_skipped_at_load_and_nev response: Final = candidate.request( "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": prompt}]} ) - assert response.status_code == 200, response.text - assert response.json()["choices"][0]["message"]["content"] == reply, response.text - assert len(policy.drain()) == 0 - assert len(upstream.drain()) == 1 + assert response.status_code == 400, response.text + assert "synthetic policy denial" in response.text, response.text + assert len(policy.drain()) == 1 + assert len(upstream.drain()) == 0 guardrails: Final = candidate.get("/v2/guardrails/list")["guardrails"] - assert all(object_value(row)["guardrail_name"] != identity for row in guardrails), guardrails + assert any(object_value(row)["guardrail_name"] == identity for row in guardrails), guardrails diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index 508736fb78e..b6330f30b2b 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -1123,7 +1123,10 @@ async def test_update_guardrail_endpoint( prisma_client=mocker.ANY, ) - mock_in_memory_handler.sync_guardrail_from_db.assert_called_once_with(guardrail=mocker.ANY) + mock_in_memory_handler.sync_guardrail_from_db.assert_called_once_with( + guardrail=mocker.ANY, + reject_invalid_logging_only_scope=True, + ) if scenario == "success_sync_fails_unexpected_error": assert mock_logger is not None @@ -1252,7 +1255,10 @@ async def test_patch_guardrail_endpoint( mock_guardrail_registry.update_guardrail_in_db.assert_called_once() - mock_in_memory_handler.sync_guardrail_from_db.assert_called_once_with(guardrail=mocker.ANY) + mock_in_memory_handler.sync_guardrail_from_db.assert_called_once_with( + guardrail=mocker.ANY, + reject_invalid_logging_only_scope=True, + ) if scenario == "success_sync_fails_unexpected_error": assert mock_logger is not None @@ -1276,6 +1282,34 @@ async def test_patch_guardrail_rejects_mcp_only_on_violation_with_422(mocker, mo mock_guardrail_registry.update_guardrail_in_db.assert_not_called() +@pytest.mark.asyncio +async def test_patch_guardrail_rejects_invalid_logging_only_scope_with_422(mocker, mock_guardrail_registry): + mocker.patch("litellm.proxy.proxy_server.prisma_client", mocker.Mock()) + mocker.patch( + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", + mock_guardrail_registry, + ) + mock_in_memory_handler = mocker.Mock(spec=InMemoryGuardrailHandler) + mock_in_memory_handler.sync_guardrail_from_db.side_effect = ValueError( + "Guardrail test-db-guardrail: logging_only_scope is set, but mode does not include logging_only" + ) + mocker.patch( + "litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", + mock_in_memory_handler, + ) + request = PatchGuardrailRequest(litellm_params=BaseLitellmParams(mode="pre_call", logging_only_scope="input")) + + with pytest.raises(HTTPException) as exc_info: + await patch_guardrail("test-guardrail-id", request, user_api_key_dict=MOCK_ADMIN_USER) + + assert exc_info.value.status_code == 422 + assert "update rejected" in str(exc_info.value.detail) + mock_in_memory_handler.sync_guardrail_from_db.assert_called_once_with( + guardrail=mocker.ANY, + reject_invalid_logging_only_scope=True, + ) + + @pytest.mark.parametrize( "scenario,expected_result,expected_exception", [ diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py index ddfdabf689d..56cfdd120f2 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py @@ -966,6 +966,8 @@ class TestLoggingOnlyScopeValidation: mode: str | list[str] | Mode, scope: LoggingOnlyScope | None, callback_type: type[CustomGuardrail] = _LoggingOnlyScopeSupportedGuardrail, + reject_invalid_logging_only_scope: bool = False, + assert_registered: bool = False, ) -> CustomGuardrail: import litellm from litellm.proxy.guardrails import guardrail_registry as registry_module @@ -980,6 +982,7 @@ class TestLoggingOnlyScopeValidation: callback: Final = callback_type( guardrail_name=guardrail["guardrail_name"], event_hook=litellm_params.mode, + default_on=True, supported_event_hooks=supported_event_hooks, ) litellm.logging_callback_manager.add_litellm_callback(callback) @@ -999,11 +1002,14 @@ class TestLoggingOnlyScopeValidation: "mode": mode, "logging_only_scope": scope, }, - } + }, + reject_invalid_logging_only_scope=reject_invalid_logging_only_scope, ) assert result is not None callback: Final = handler.guardrail_id_to_custom_guardrail[result["guardrail_id"]] assert callback is not None + if assert_registered: + assert callback in lists[0] return callback except ValueError: callback: Final = created_callbacks[0] @@ -1014,9 +1020,15 @@ class TestLoggingOnlyScopeValidation: callback_list[:] = snapshot registry_module.guardrail_initializer_registry.pop(guardrail_type, None) - def test_scope_requires_logging_only_mode(self) -> None: + def test_scope_without_logging_only_mode_is_ignored_at_load(self) -> None: + callback: Final = self._initialize(mode="pre_call", scope="input", assert_registered=True) + + assert callback.logging_only_scope is None + assert callback.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is True + + def test_scope_without_logging_only_mode_is_rejected_for_api_writes(self) -> None: with pytest.raises(ValueError, match="logging_only_scope is set") as exc_info: - self._initialize(mode="pre_call", scope="input") + self._initialize(mode="pre_call", scope="input", reject_invalid_logging_only_scope=True) assert str(exc_info.value) == ( "Guardrail logging-only-scope-guardrail: logging_only_scope is set, but mode does not include " @@ -1036,12 +1048,23 @@ class TestLoggingOnlyScopeValidation: assert callback.logging_only_scope == "input" - def test_directional_scope_rejected_when_guardrail_owns_logging_hook(self) -> None: + def test_directional_scope_is_ignored_at_load_when_guardrail_owns_logging_hook(self) -> None: + callback: Final = self._initialize( + mode="logging_only", + scope="input", + callback_type=_LoggingOnlyScopeUnsupportedGuardrail, + assert_registered=True, + ) + + assert callback.logging_only_scope is None + + def test_directional_scope_rejected_for_api_writes_when_guardrail_owns_logging_hook(self) -> None: with pytest.raises(ValueError, match="logging_only_scope='input' is not supported") as exc_info: self._initialize( mode="logging_only", scope="input", callback_type=_LoggingOnlyScopeUnsupportedGuardrail, + reject_invalid_logging_only_scope=True, ) assert str(exc_info.value) == ( From 089608554df2e2edb2c1d0d6954f9f138cef0167 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 10:08:22 +0000 Subject: [PATCH 09/20] chore(ui): avoid inline guardrail test fixtures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrail_info.characterization.test.tsx | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.characterization.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.characterization.test.tsx index 3918e6ccf35..efbcc10e3ab 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.characterization.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.characterization.test.tsx @@ -165,24 +165,19 @@ describe("GuardrailInfoView update payload characterization", () => { }); it("shows and updates the logging-only scope", async () => { - vi.mocked(networking.getGuardrailInfo).mockResolvedValue( - guardrail({ - guardrailIdentifier: "gr-abc", - api_key: "sk-old", - mode: "logging_only", - logging_only_scope: "input", - }), - ); + const guardrailParams = { + guardrailIdentifier: "gr-abc", + api_key: "sk-old", + mode: "logging_only", + logging_only_scope: "input", + }; + vi.mocked(networking.getGuardrailInfo).mockResolvedValue(guardrail(guardrailParams)); const user = userEvent.setup({ delay: null }); renderView(); expect(await screen.findAllByText("Input only (request)")).toHaveLength(2); await openEditor(user); - await chooseSelectOption( - user, - screen.getByLabelText("Logging only scope"), - "Output only (response)", - ); + await chooseSelectOption(user, screen.getByLabelText("Logging only scope"), "Output only (response)"); await saveChanges(user); await waitFor(() => expect(networking.updateGuardrailCall).toHaveBeenCalledTimes(1)); @@ -190,14 +185,13 @@ describe("GuardrailInfoView update payload characterization", () => { }); it("clears the logging-only scope when the edit choice returns to default", async () => { - vi.mocked(networking.getGuardrailInfo).mockResolvedValue( - guardrail({ - guardrailIdentifier: "gr-abc", - api_key: "sk-old", - mode: "logging_only", - logging_only_scope: "input", - }), - ); + const guardrailParams = { + guardrailIdentifier: "gr-abc", + api_key: "sk-old", + mode: "logging_only", + logging_only_scope: "input", + }; + vi.mocked(networking.getGuardrailInfo).mockResolvedValue(guardrail(guardrailParams)); const user = userEvent.setup({ delay: null }); renderView(); await openEditor(user); From 6cbb57a7973c2f20b6a9b2e23fe0fdeff8fb223f Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 10:28:09 +0000 Subject: [PATCH 10/20] fix(guardrails): support directional scope in PATCH and provider UI Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/custom_guardrail.py | 13 +- .../proxy/guardrails/guardrail_endpoints.py | 49 +++-- .../proxy/guardrails/guardrail_registry.py | 4 +- litellm/types/guardrails.py | 1 + .../guardrails/test_guardrail_endpoints.py | 179 +++++++++++++++++- .../integrations/test_custom_guardrail.py | 4 +- .../_components/GuardrailFormField.tsx | 36 ++-- ...d_guardrail_form.characterization.test.tsx | 24 +++ .../_components/add_guardrail_form.tsx | 9 +- .../guardrail_info.characterization.test.tsx | 1 + .../guardrails/_components/guardrail_info.tsx | 5 + .../guardrail_info_helpers.test.tsx | 42 ++++ .../_components/guardrail_info_helpers.tsx | 26 +++ 13 files changed, 348 insertions(+), 45 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 42e103f942b..8e71f7a08ff 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -806,10 +806,11 @@ class CustomGuardrail(CustomLogger): def uses_apply_guardrail_interface(self) -> bool: return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail - def supports_logging_only_scope(self) -> bool: + @classmethod + def supports_logging_only_scope(cls) -> bool: return ( - self.uses_apply_guardrail_interface() - and type(self).async_logging_hook is CustomGuardrail.async_logging_hook + cls.apply_guardrail is not CustomGuardrail.apply_guardrail + and cls.async_logging_hook is CustomGuardrail.async_logging_hook ) def _deployment_hook_target(self) -> "CustomLogger": @@ -992,13 +993,13 @@ class CustomGuardrail(CustomLogger): def _copy_scratch_request_fields( self, - kwargs: dict, # mutable-ok: CustomLogger.async_logging_hook contract + kwargs: Mapping[str, object], ) -> tuple[object, object]: - optional_params: Final = kwargs.get("optional_params") or {} + optional_params: Final = kwargs.get("optional_params") try: return ( copy.deepcopy(kwargs.get("messages") or kwargs.get("input")), - copy.deepcopy(optional_params.get("tools")), + copy.deepcopy(optional_params.get("tools") if isinstance(optional_params, Mapping) else None), ) except Exception: if self.logging_only_scope == "output": diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index d52996cd68d..c3ed2fd8cc9 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -32,7 +32,7 @@ from litellm.proxy.guardrails.guardrail_hooks.custom_code.sandbox import ( build_sandbox_globals, compile_sandboxed, ) -from litellm.proxy.guardrails.guardrail_registry import GuardrailRegistry +from litellm.proxy.guardrails.guardrail_registry import GuardrailRegistry, _configured_event_hooks from litellm.proxy.guardrails.usage_endpoints import router as guardrails_usage_router from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view from litellm.repositories.prisma_protocols import TableActions @@ -1212,19 +1212,30 @@ async def patch_guardrail( # Update litellm_params if default_on is provided or pii_entities_config is provided existing_litellm_params: Final = _as_str_object_mapping(dict(existing_guardrail.get("litellm_params", {}))) - litellm_params = LitellmParams(**existing_litellm_params) - if request.litellm_params is not None: - requested_litellm_params: Final = request.litellm_params.model_dump(exclude_unset=True) - litellm_params_dict: Final = litellm_params.model_dump(exclude_unset=True) - litellm_params_dict.update(requested_litellm_params) - merged_litellm_params: Final = _as_str_object_mapping(litellm_params_dict) - try: - litellm_params = LitellmParams(**merged_litellm_params) - except ValidationError as validation_error: - raise HTTPException( - status_code=422, - detail=f"Invalid guardrail configuration, update rejected: {validation_error}", - ) from validation_error + current_litellm_params: Final = LitellmParams(**existing_litellm_params) + requested_litellm_params: Final = ( + request.litellm_params.model_dump(exclude_unset=True) if request.litellm_params is not None else {} + ) + merged_litellm_params: Final = _as_str_object_mapping( + {**current_litellm_params.model_dump(exclude_unset=True), **requested_litellm_params} + ) + try: + parsed_litellm_params: Final = LitellmParams(**merged_litellm_params) + except ValidationError as validation_error: + raise HTTPException( + status_code=422, + detail=f"Invalid guardrail configuration, update rejected: {validation_error}", + ) from validation_error + clear_stored_scope: Final = ( + "logging_only_scope" not in requested_litellm_params + and parsed_litellm_params.logging_only_scope is not None + and GuardrailEventHooks.logging_only.value not in _configured_event_hooks(parsed_litellm_params.mode) + ) + litellm_params: Final = ( + LitellmParams(**{**merged_litellm_params, "logging_only_scope": None}) + if clear_stored_scope + else parsed_litellm_params + ) # Update guardrail_info if provided guardrail_info: Final = ( @@ -1253,7 +1264,7 @@ async def patch_guardrail( try: IN_MEMORY_GUARDRAIL_HANDLER.sync_guardrail_from_db( guardrail=guardrail, - reject_invalid_logging_only_scope=True, + reject_invalid_logging_only_scope="logging_only_scope" in requested_litellm_params, ) verbose_proxy_logger.info( "Immediate sync: Successfully updated guardrail '%s' (ID: %s)", guardrail_name, guardrail_id @@ -1398,7 +1409,7 @@ async def get_guardrail_info(guardrail_id: str): tags=["Guardrails"], dependencies=[Depends(user_api_key_auth)], ) -async def get_guardrail_ui_settings(): +async def get_guardrail_ui_settings() -> GuardrailUIAddGuardrailSettings: """ Get the UI settings for the guardrails @@ -1432,12 +1443,18 @@ async def get_guardrail_ui_settings(): # above; it only runs on pre_call. {SupportedGuardrailIntegrations.HIDE_SECRETS.value: [GuardrailEventHooks.pre_call.value]} ) + providers_without_directional_logging_only_scope: Final = [ + provider + for provider, guardrail_class in guardrail_class_registry.items() + if not guardrail_class.supports_logging_only_scope() + ] return GuardrailUIAddGuardrailSettings( supported_entities=[entity.value for entity in PiiEntityType], supported_actions=[action.value for action in PiiAction], supported_modes=[mode.value for mode in GuardrailEventHooks], supported_modes_by_provider=supported_modes_by_provider, + providers_without_directional_logging_only_scope=providers_without_directional_logging_only_scope, pii_entity_categories=category_maps, content_filter_settings={ "prebuilt_patterns": get_pattern_metadata(), diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index 5f071083683..ea06f4993b1 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -711,7 +711,7 @@ class InMemoryGuardrailHandler: previous instance and raises), anything else only refreshes the stored row """ updated_guardrail: Final = cast(Guardrail, {**guardrail, "guardrail_id": guardrail_id}) - if self._has_guardrail_params_changed(guardrail_id, updated_guardrail): + if reject_invalid_logging_only_scope or self._has_guardrail_params_changed(guardrail_id, updated_guardrail): self.reinitialize_guardrail( guardrail=updated_guardrail, source=source, @@ -936,7 +936,7 @@ class InMemoryGuardrailHandler: verbose_proxy_logger.error("Cannot sync guardrail without guardrail_id") return None - if self._has_guardrail_params_changed(guardrail_id, guardrail): + if reject_invalid_logging_only_scope or self._has_guardrail_params_changed(guardrail_id, guardrail): guardrail_name: Final = guardrail.get("guardrail_name", "Unknown") verbose_proxy_logger.info( "Guardrail '%s' (ID: %s) params changed, re-initializing...", guardrail_name, guardrail_id diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index a98446fff43..e4499248500 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -1314,6 +1314,7 @@ class GuardrailUIAddGuardrailSettings(BaseModel): supported_actions: list[str] supported_modes: list[str] supported_modes_by_provider: dict[str, list[str]] + providers_without_directional_logging_only_scope: list[str] pii_entity_categories: list[PiiEntityCategoryMap] content_filter_settings: dict[str, object] | None = None diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index b6330f30b2b..12310f5d67c 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -1,5 +1,6 @@ import json import time +from collections.abc import Callable from datetime import datetime from typing import Dict, List, Optional from unittest.mock import AsyncMock @@ -9,6 +10,7 @@ import pytest from fastapi import HTTPException +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.guardrails.guardrail_endpoints import ( CreateGuardrailRequest, @@ -42,10 +44,12 @@ from litellm.proxy.guardrails.guardrail_registry import ( from litellm.types.guardrails import ( ApplyGuardrailRequest, BaseLitellmParams, + GuardrailEventHooks, Guardrail, GuardrailInfoResponse, LitellmParams, ) +from litellm.types.utils import GenericGuardrailAPIInputs # Mock data for testing MOCK_DB_GUARDRAIL = { @@ -85,6 +89,44 @@ MOCK_PATCH_REQUEST = PatchGuardrailRequest( ) +class _PatchScopeSupportedGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: str, + logging_obj: object | None = None, + ) -> GenericGuardrailAPIInputs: + return inputs + + +class _PatchScopeUnsupportedGuardrail(_PatchScopeSupportedGuardrail): + async def async_logging_hook( + self, + kwargs: dict[str, object], + result: object, + call_type: str, + ) -> tuple[dict[str, object], object]: + return kwargs, result + + +def _patch_scope_initializer( + callback_type: type[CustomGuardrail], +) -> Callable[[LitellmParams, Guardrail], CustomGuardrail]: + def _initializer(litellm_params: LitellmParams, guardrail: Guardrail) -> CustomGuardrail: + import litellm + + callback = callback_type( + guardrail_name=guardrail["guardrail_name"], + event_hook=litellm_params.mode, + default_on=litellm_params.default_on, + ) + litellm.logging_callback_manager.add_litellm_callback(callback) + return callback + + return _initializer + + @pytest.fixture def mock_prisma_client(mocker): """Mock Prisma client for testing""" @@ -124,6 +166,37 @@ def mock_guardrail_registry(mocker): return mock_registry +def _setup_patch_scope_guardrail( + mocker, + monkeypatch, + mock_guardrail_registry, + callback_type: type[CustomGuardrail], + guardrail_type: str, + litellm_params: dict[str, object], +) -> tuple[InMemoryGuardrailHandler, Guardrail]: + from litellm.proxy.guardrails import guardrail_registry as registry_module + + guardrail: Guardrail = { + "guardrail_id": "patch-scope-test", + "guardrail_name": "Patch scope test", + "litellm_params": {"guardrail": guardrail_type, **litellm_params}, + "guardrail_info": {}, + } + mock_guardrail_registry.get_guardrail_by_id_from_db.return_value = guardrail + mock_guardrail_registry.update_guardrail_in_db.return_value = guardrail + monkeypatch.setitem( + registry_module.guardrail_initializer_registry, + guardrail_type, + _patch_scope_initializer(callback_type), + ) + handler = InMemoryGuardrailHandler() + handler.initialize_guardrail(guardrail=guardrail, source="db") + mocker.patch("litellm.proxy.proxy_server.prisma_client", mocker.Mock()) + mocker.patch("litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", mock_guardrail_registry) + mocker.patch("litellm.proxy.guardrails.guardrail_registry.IN_MEMORY_GUARDRAIL_HANDLER", handler) + return handler, guardrail + + @pytest.mark.asyncio async def test_list_guardrails_v2_with_db_and_config(mocker, mock_prisma_client, mock_in_memory_handler): """Test listing guardrails from both DB and config""" @@ -1257,7 +1330,7 @@ async def test_patch_guardrail_endpoint( mock_in_memory_handler.sync_guardrail_from_db.assert_called_once_with( guardrail=mocker.ANY, - reject_invalid_logging_only_scope=True, + reject_invalid_logging_only_scope=False, ) if scenario == "success_sync_fails_unexpected_error": @@ -1310,6 +1383,103 @@ async def test_patch_guardrail_rejects_invalid_logging_only_scope_with_422(mocke ) +@pytest.mark.asyncio +async def test_patch_guardrail_clears_scope_when_logging_only_mode_is_removed( + mocker, monkeypatch, mock_guardrail_registry +): + handler, stored_guardrail = _setup_patch_scope_guardrail( + mocker, + monkeypatch, + mock_guardrail_registry, + _PatchScopeSupportedGuardrail, + "patch_scope_supported_test", + { + "mode": ["pre_call", "logging_only"], + "logging_only_scope": "output", + "default_on": True, + }, + ) + request = PatchGuardrailRequest(litellm_params=BaseLitellmParams(mode=["pre_call"])) + + try: + result = await patch_guardrail( + stored_guardrail["guardrail_id"], + request, + user_api_key_dict=MOCK_ADMIN_USER, + ) + + assert result["guardrail_id"] == stored_guardrail["guardrail_id"] + persisted_guardrail = mock_guardrail_registry.update_guardrail_in_db.call_args.kwargs["guardrail"] + assert persisted_guardrail["litellm_params"].logging_only_scope is None + callback = handler.guardrail_id_to_custom_guardrail[stored_guardrail["guardrail_id"]] + assert callback.logging_only_scope is None + assert callback.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is True + finally: + handler.delete_in_memory_guardrail(stored_guardrail["guardrail_id"]) + + +@pytest.mark.asyncio +async def test_patch_guardrail_tolerates_stored_unsupported_scope_on_unrelated_update( + mocker, monkeypatch, mock_guardrail_registry, caplog +): + handler, stored_guardrail = _setup_patch_scope_guardrail( + mocker, + monkeypatch, + mock_guardrail_registry, + _PatchScopeUnsupportedGuardrail, + "patch_scope_unsupported_test", + {"mode": "logging_only", "logging_only_scope": "output", "default_on": True}, + ) + caplog.clear() + request = PatchGuardrailRequest(litellm_params=BaseLitellmParams(default_on=False)) + + try: + result = await patch_guardrail( + stored_guardrail["guardrail_id"], + request, + user_api_key_dict=MOCK_ADMIN_USER, + ) + + assert result["guardrail_id"] == stored_guardrail["guardrail_id"] + callback = handler.guardrail_id_to_custom_guardrail[stored_guardrail["guardrail_id"]] + assert callback.logging_only_scope is None + assert any("Ignoring logging_only_scope" in record.getMessage() for record in caplog.records) + finally: + handler.delete_in_memory_guardrail(stored_guardrail["guardrail_id"]) + + +@pytest.mark.asyncio +async def test_patch_guardrail_rejects_explicit_unsupported_scope_and_rolls_back( + mocker, monkeypatch, mock_guardrail_registry +): + handler, stored_guardrail = _setup_patch_scope_guardrail( + mocker, + monkeypatch, + mock_guardrail_registry, + _PatchScopeUnsupportedGuardrail, + "patch_scope_unsupported_test", + {"mode": "logging_only", "logging_only_scope": "output", "default_on": True}, + ) + request = PatchGuardrailRequest(litellm_params=BaseLitellmParams(logging_only_scope="output")) + + try: + with pytest.raises(HTTPException) as exc_info: + await patch_guardrail( + stored_guardrail["guardrail_id"], + request, + user_api_key_dict=MOCK_ADMIN_USER, + ) + + assert exc_info.value.status_code == 422 + assert mock_guardrail_registry.update_guardrail_in_db.call_count == 2 + restored_guardrail = mock_guardrail_registry.update_guardrail_in_db.call_args_list[-1].kwargs["guardrail"] + assert restored_guardrail["litellm_params"].logging_only_scope == "output" + callback = handler.guardrail_id_to_custom_guardrail[stored_guardrail["guardrail_id"]] + assert callback.logging_only_scope is None + finally: + handler.delete_in_memory_guardrail(stored_guardrail["guardrail_id"]) + + @pytest.mark.parametrize( "scenario,expected_result,expected_exception", [ @@ -2495,6 +2665,13 @@ async def test_ui_settings_map_matches_runtime_supported_event_hooks(): from litellm.proxy.guardrails.guardrail_registry import guardrail_class_registry result = await get_guardrail_ui_settings() + expected_without_directional_scope = { + provider + for provider, guardrail_class in guardrail_class_registry.items() + if not guardrail_class.supports_logging_only_scope() + } + assert set(result.providers_without_directional_logging_only_scope) == expected_without_directional_scope + assert "xecguard" in result.providers_without_directional_logging_only_scope for provider, guardrail_class in guardrail_class_registry.items(): declared = guardrail_class.get_supported_event_hooks() diff --git a/tests/unit/integrations/test_custom_guardrail.py b/tests/unit/integrations/test_custom_guardrail.py index c31379e7f1d..052e3230e0a 100644 --- a/tests/unit/integrations/test_custom_guardrail.py +++ b/tests/unit/integrations/test_custom_guardrail.py @@ -2996,9 +2996,7 @@ async def test_native_lifecycle_guardrail_logging_only_scans_assembled_response( from litellm.types.utils import Choices, Message, ModelResponse guardrail = _NativeLifecycleLoggingGuardrail() - assembled = ModelResponse( - choices=[Choices(message=Message(role="assistant", content="assembled stream text"))] - ) + assembled = ModelResponse(choices=[Choices(message=Message(role="assistant", content="assembled stream text"))]) sentinel_result = object() kwargs = { "model": "gpt-5.4-mini", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx index 53a624d4422..3b76c9e8016 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx @@ -6,7 +6,11 @@ import { useController, type Control, type ControllerRenderProps, type RegisterO import { Field, FieldDescription, FieldError, FieldLabel } from "@/components/ui/field"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { modeIncludesLoggingOnly, type LoggingOnlyScopeChoice } from "./guardrail_info_helpers"; +import { + getLoggingOnlyScopeOptions, + modeIncludesLoggingOnly, + type LoggingOnlyScopeChoice, +} from "./guardrail_info_helpers"; export interface GuardrailCriterion { name: string; @@ -126,23 +130,20 @@ export const SkipMessageSelect: React.FC<{ control: GuardrailFieldControlProps } ); }; -const LOGGING_ONLY_SCOPE_ITEMS: Array<{ label: string; value: LoggingOnlyScopeChoice }> = [ - { label: "Default (request and response)", value: "default" }, - { label: "Input only (request)", value: "input" }, - { label: "Output only (response)", value: "output" }, - { label: "Both (request and response)", value: "both" }, -]; - -export const LoggingOnlyScopeSelect: React.FC<{ control: GuardrailFieldControlProps }> = ({ control }) => { +export const LoggingOnlyScopeSelect: React.FC<{ + control: GuardrailFieldControlProps; + directionalScopeSupported: boolean; +}> = ({ control, directionalScopeSupported }) => { const { id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy } = control; + const items = getLoggingOnlyScopeOptions(directionalScopeSupported); return ( - - {LOGGING_ONLY_SCOPE_ITEMS.map((item) => ( + {items.map((item) => ( {item.label} @@ -152,10 +153,11 @@ export const LoggingOnlyScopeSelect: React.FC<{ control: GuardrailFieldControlPr ); }; -export const LoggingOnlyScopeField: React.FC<{ control: GuardrailFormControl; mode: unknown }> = ({ - control, - mode, -}) => { +export const LoggingOnlyScopeField: React.FC<{ + control: GuardrailFormControl; + mode: unknown; + directionalScopeSupported: boolean; +}> = ({ control, mode, directionalScopeSupported }) => { if (!modeIncludesLoggingOnly(mode)) return null; return ( @@ -167,7 +169,9 @@ export const LoggingOnlyScopeField: React.FC<{ control: GuardrailFormControl; mo "Which direction a logging_only scan observes. Observe-only scans never block; pre_call and post_call on this guardrail still block.", )} > - {(fieldControl) => } + {(fieldControl) => ( + + )} ); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx index 89a00b08d6b..2b6b67ab440 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx @@ -37,6 +37,7 @@ const uiSettings = { supported_entities: [], supported_actions: [], supported_modes: ["pre_call", "post_call"], + providers_without_directional_logging_only_scope: [], pii_entity_categories: [], }; @@ -131,6 +132,29 @@ describe("AddGuardrailForm create payload characterization", () => { expect(payload()?.litellm_params.logging_only_scope).toBe("output"); }); + it("hides directional scope choices for providers that do not support them", async () => { + vi.mocked(networking.getGuardrailUISettings).mockResolvedValue({ + ...uiSettings, + supported_modes: ["pre_call", "logging_only"], + providers_without_directional_logging_only_scope: ["xecguard"], + }); + vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({ + ...providerParams, + xecguard: { ui_friendly_name: "XecGuard" }, + }); + const user = userEvent.setup({ delay: null }); + renderForm(); + + await pickProvider(user, "XecGuard"); + await user.click(screen.getByLabelText("Mode")); + await user.click((await screen.findAllByText("logging_only")).at(-1) as HTMLElement); + await user.click(await screen.findByLabelText("Logging only scope")); + + expect(screen.queryByRole("option", { name: "Input only (request)" })).not.toBeInTheDocument(); + expect(screen.queryByRole("option", { name: "Output only (response)" })).not.toBeInTheDocument(); + expect(screen.getByRole("option", { name: "Both (request and response)" })).toBeInTheDocument(); + }); + it("hides logging-only scope and omits it from a pre-call payload", async () => { const user = userEvent.setup({ delay: null }); renderForm(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx index 689dbbfefa2..4c3f9ce693e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx @@ -23,6 +23,7 @@ import { shouldRenderContentFilterConfigSettings, shouldRenderLLMJudgeFields, shouldRenderPIIConfigSettings, + supportsDirectionalLoggingOnlyScope, toModeArray, type LoggingOnlyScope, type LoggingOnlyScopeChoice, @@ -95,6 +96,7 @@ interface GuardrailSettings { supported_actions: string[]; supported_modes: string[]; supported_modes_by_provider?: Record; + providers_without_directional_logging_only_scope?: string[]; pii_entity_categories: Array<{ category: string; entities: string[]; @@ -688,6 +690,7 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a const providerLabels: Record = getGuardrailProviders(); const providerKeys = Object.keys(providerLabels); const supportedModes = getSupportedModesForProvider(guardrailSettings, selectedProvider) ?? DEFAULT_MODES; + const directionalScopeSupported = supportsDirectionalLoggingOnlyScope(guardrailSettings, selectedProvider); return ( = ({ visible, onClose, a {(fieldControl) => } - + {/* Use the GuardrailProviderFields component to render provider-specific fields */} {showProviderFields && ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.characterization.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.characterization.test.tsx index efbcc10e3ab..460a01ea6c9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.characterization.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.characterization.test.tsx @@ -25,6 +25,7 @@ const uiSettings = { supported_actions: [], pii_entity_categories: [], supported_modes: ["pre_call", "post_call"], + providers_without_directional_logging_only_scope: [], }; const bedrockParams = { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx index e75e55244de..ca19abcc687 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx @@ -41,6 +41,7 @@ import { loggingOnlyScopeToChoice, skipSystemMessageToChoice, skipToolMessageToChoice, + supportsDirectionalLoggingOnlyScope, type SkipSystemMessageChoice, type SkipToolMessageChoice, } from "./guardrail_info_helpers"; @@ -84,6 +85,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, entities: string[]; }>; supported_modes: string[]; + providers_without_directional_logging_only_scope?: string[]; content_filter_settings?: { prebuilt_patterns: Array<{ name: string; @@ -112,6 +114,8 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, const [toolPermissionConfig, setToolPermissionConfig] = useState(emptyToolPermissionConfig); const [toolPermissionDirty, setToolPermissionDirty] = useState(false); const [customCodeModalVisible, setCustomCodeModalVisible] = useState(false); + const guardrailProvider = guardrailData?.litellm_params?.guardrail ?? null; + const directionalScopeSupported = supportsDirectionalLoggingOnlyScope(guardrailSettings, guardrailProvider); // Content Filter data ref (managed by ContentFilterManager) const contentFilterDataRef = React.useRef<{ @@ -742,6 +746,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, {guardrailData.litellm_params?.guardrail === "presidio" && ( <> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx index 95c66e40d6b..de1be697221 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx @@ -18,8 +18,10 @@ import { loggingOnlyScopeToChoice, choiceToLoggingOnlyScope, getLoggingOnlyScopeUpdate, + getLoggingOnlyScopeOptions, formatLoggingOnlyScope, modeIncludesLoggingOnly, + supportsDirectionalLoggingOnlyScope, } from "./guardrail_info_helpers"; describe("guardrail_info_helpers", () => { @@ -32,6 +34,7 @@ describe("guardrail_info_helpers", () => { "PresidioPII", "Bedrock", "Lakera", + "Xecguard", "LitellmContentFilter", "ToolPermission", "BlockCodeExecution", @@ -288,6 +291,45 @@ describe("guardrail_info_helpers", () => { ).toBe(true); expect(modeIncludesLoggingOnly("pre_call")).toBe(false); }); + + it("filters directional options for unsupported providers and keeps all options otherwise", () => { + expect(getLoggingOnlyScopeOptions(false).map((option) => option.value)).toEqual(["default", "both"]); + expect(getLoggingOnlyScopeOptions(true).map((option) => option.value)).toEqual([ + "default", + "input", + "output", + "both", + ]); + + expect( + supportsDirectionalLoggingOnlyScope( + { providers_without_directional_logging_only_scope: ["xecguard"] }, + "Xecguard", + ), + ).toBe(false); + expect( + supportsDirectionalLoggingOnlyScope( + { providers_without_directional_logging_only_scope: ["xecguard"] }, + "xecguard", + ), + ).toBe(false); + expect( + supportsDirectionalLoggingOnlyScope( + { providers_without_directional_logging_only_scope: ["xecguard"] }, + "Bedrock", + ), + ).toBe(true); + expect( + supportsDirectionalLoggingOnlyScope( + { providers_without_directional_logging_only_scope: ["xecguard"] }, + "unknown-provider", + ), + ).toBe(true); + expect(supportsDirectionalLoggingOnlyScope(null, "Xecguard")).toBe(true); + expect(getLoggingOnlyScopeOptions(supportsDirectionalLoggingOnlyScope(null, "Xecguard"))).toEqual( + getLoggingOnlyScopeOptions(true), + ); + }); }); describe("skipSystemMessageToChoice / choiceToSkipSystemForCreate", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx index 95d463a55a9..8bf9a26a16b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx @@ -116,6 +116,14 @@ export const toModeArray = (raw: unknown): string[] => { export type LoggingOnlyScope = "input" | "output" | "both"; export type LoggingOnlyScopeChoice = "default" | LoggingOnlyScope; +export type LoggingOnlyScopeOption = { label: string; value: LoggingOnlyScopeChoice }; + +const LOGGING_ONLY_SCOPE_OPTIONS: LoggingOnlyScopeOption[] = [ + { label: "Default (request and response)", value: "default" }, + { label: "Input only (request)", value: "input" }, + { label: "Output only (response)", value: "output" }, + { label: "Both (request and response)", value: "both" }, +]; export const loggingOnlyScopeToChoice = (v: string | null | undefined): LoggingOnlyScopeChoice => v === "input" || v === "output" || v === "both" ? v : "default"; @@ -150,6 +158,24 @@ export const modeIncludesLoggingOnly = (raw: unknown): boolean => { return toModeArray(fallback).includes("logging_only") || taggedModes; }; +export const getLoggingOnlyScopeOptions = (directionalScopeSupported: boolean): LoggingOnlyScopeOption[] => + directionalScopeSupported + ? LOGGING_ONLY_SCOPE_OPTIONS + : LOGGING_ONLY_SCOPE_OPTIONS.filter((option) => option.value === "default" || option.value === "both"); + +export const supportsDirectionalLoggingOnlyScope = ( + settings: { providers_without_directional_logging_only_scope?: string[] } | null, + selectedProvider: string | null, +): boolean => { + const providerKey = selectedProvider + ? (guardrail_provider_map[selectedProvider] ?? + Object.values(guardrail_provider_map).find( + (value) => value.toLowerCase() === selectedProvider.toLowerCase(), + ))?.toLowerCase() + : null; + return !providerKey || !settings?.providers_without_directional_logging_only_scope?.includes(providerKey); +}; + export const formatGuardrailMode = (raw: unknown): string => { const flat: string[] = toModeArray(raw); if (flat.length > 0) return flat.join(", "); From 6388bdf73eb4ded04873211a21efe7cd31103db4 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 10:33:48 +0000 Subject: [PATCH 11/20] fix(ui): format guardrail files for frontend lint Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrails/_components/GuardrailModeDisplay.tsx | 6 +----- .../add_guardrail_form.characterization.test.tsx | 6 +----- .../(dashboard)/guardrails/_components/guardrail_info.tsx | 3 ++- .../guardrails/_components/guardrail_info_helpers.tsx | 8 ++++---- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailModeDisplay.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailModeDisplay.tsx index fe45e183e82..e5fa1669a44 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailModeDisplay.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailModeDisplay.tsx @@ -1,11 +1,7 @@ import React from "react"; import { Badge } from "@/components/ui/badge"; import { Card } from "@/components/ui/card"; -import { - formatGuardrailMode, - formatLoggingOnlyScope, - modeIncludesLoggingOnly, -} from "./guardrail_info_helpers"; +import { formatGuardrailMode, formatLoggingOnlyScope, modeIncludesLoggingOnly } from "./guardrail_info_helpers"; type GuardrailModeParams = { mode?: unknown; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx index 2b6b67ab440..230be13ef33 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx @@ -118,11 +118,7 @@ describe("AddGuardrailForm create payload characterization", () => { await pickProvider(user, "Bedrock Guardrail"); await user.click(screen.getByLabelText("Mode")); await user.click((await screen.findAllByText("logging_only")).at(-1) as HTMLElement); - await chooseSelectOption( - user, - await screen.findByLabelText("Logging only scope"), - "Output only (response)", - ); + await chooseSelectOption(user, await screen.findByLabelText("Logging only scope"), "Output only (response)"); await user.type(await screen.findByPlaceholderText("The guardrail id on Bedrock"), "gr-123"); await user.click(screen.getByRole("button", { name: "Next" })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx index ca19abcc687..73d9fb8123e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx @@ -226,7 +226,8 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, if (!guardrailData) return; form.setValue("guardrail_name", guardrailData.guardrail_name); form.setValue("default_on", guardrailData.litellm_params?.default_on); - form.setValue("logging_only_scope_choice", loggingOnlyScopeToChoice(guardrailData.litellm_params?.logging_only_scope)); + const storedLoggingOnlyScope = guardrailData.litellm_params?.logging_only_scope; + form.setValue("logging_only_scope_choice", loggingOnlyScopeToChoice(storedLoggingOnlyScope)); form.setValue( "skip_system_message_choice", skipSystemMessageToChoice(guardrailData.litellm_params?.skip_system_message_in_guardrail), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx index 8bf9a26a16b..da28e0962a8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx @@ -168,10 +168,10 @@ export const supportsDirectionalLoggingOnlyScope = ( selectedProvider: string | null, ): boolean => { const providerKey = selectedProvider - ? (guardrail_provider_map[selectedProvider] ?? - Object.values(guardrail_provider_map).find( - (value) => value.toLowerCase() === selectedProvider.toLowerCase(), - ))?.toLowerCase() + ? ( + guardrail_provider_map[selectedProvider] ?? + Object.values(guardrail_provider_map).find((value) => value.toLowerCase() === selectedProvider.toLowerCase()) + )?.toLowerCase() : null; return !providerKey || !settings?.providers_without_directional_logging_only_scope?.includes(providerKey); }; From bc423af7348eac1800fa25c825efdccd97e1631b Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 10:47:31 +0000 Subject: [PATCH 12/20] fix(ui): reset unsupported directional scope selections Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...d_guardrail_form.characterization.test.tsx | 30 +++++++++++++++- .../_components/add_guardrail_form.tsx | 9 ++++- .../guardrail_info.characterization.test.tsx | 17 +++++++++ .../guardrails/_components/guardrail_info.tsx | 35 ++++++++++--------- 4 files changed, 72 insertions(+), 19 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx index 230be13ef33..040d04b7f48 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { screen, waitFor } from "@testing-library/react"; +import { fireEvent, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { chooseSelectOption, renderWithProviders } from "@/../tests/test-utils"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -151,6 +151,34 @@ describe("AddGuardrailForm create payload characterization", () => { expect(screen.getByRole("option", { name: "Both (request and response)" })).toBeInTheDocument(); }); + it("clears a selected directional scope when switching to an unsupported provider", async () => { + vi.mocked(networking.getGuardrailUISettings).mockResolvedValue({ + ...uiSettings, + supported_modes: ["pre_call", "logging_only"], + providers_without_directional_logging_only_scope: ["xecguard"], + }); + vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({ + ...providerParams, + xecguard: { ui_friendly_name: "XecGuard" }, + }); + const user = userEvent.setup({ delay: null }); + renderForm(); + + fireEvent.change(await screen.findByLabelText("Guardrail Name"), { target: { value: "switch-scope" } }); + await pickProvider(user, "Bedrock Guardrail"); + await user.click(screen.getByLabelText("Mode")); + await user.click((await screen.findAllByText("logging_only")).at(-1) as HTMLElement); + await chooseSelectOption(user, await screen.findByLabelText("Logging only scope"), "Output only (response)"); + expect(screen.getByLabelText("Logging only scope")).toHaveTextContent("Output only (response)"); + await pickProvider(user, "XecGuard"); + await user.click(screen.getByRole("button", { name: "Next" })); + await user.click(await screen.findByRole("button", { name: "Create Guardrail" })); + + await waitFor(() => expect(networking.createGuardrailCall).toHaveBeenCalledTimes(1)); + expect(payload()?.litellm_params.mode).toContain("logging_only"); + expect(payload()?.litellm_params).not.toHaveProperty("logging_only_scope"); + }); + it("hides logging-only scope and omits it from a pre-call payload", async () => { const user = userEvent.setup({ delay: null }); renderForm(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx index 4c3f9ce693e..d99a31fedf0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx @@ -243,6 +243,14 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a const providerValue = guardrail_provider_map[selectedProvider]; return (providerValue || "").toLowerCase() === "tool_permission"; }, [selectedProvider]); + const directionalScopeSupported = supportsDirectionalLoggingOnlyScope(guardrailSettings, selectedProvider); + + useEffect(() => { + const scopeChoice = form.getValues("logging_only_scope_choice"); + if (!directionalScopeSupported && (scopeChoice === "input" || scopeChoice === "output")) { + form.setValue("logging_only_scope_choice", "default"); + } + }, [directionalScopeSupported, form]); // Fetch guardrail UI settings + provider params on mount / accessToken change useEffect(() => { @@ -690,7 +698,6 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a const providerLabels: Record = getGuardrailProviders(); const providerKeys = Object.keys(providerLabels); const supportedModes = getSupportedModesForProvider(guardrailSettings, selectedProvider) ?? DEFAULT_MODES; - const directionalScopeSupported = supportsDirectionalLoggingOnlyScope(guardrailSettings, selectedProvider); return ( { expect(lastPayload()).toEqual({ litellm_params: { logging_only_scope: null } }); }); + it("clears a stored directional scope for a provider that does not support it", async () => { + vi.mocked(networking.getGuardrailUISettings).mockResolvedValue({ + ...uiSettings, + providers_without_directional_logging_only_scope: ["bedrock"], + }); + vi.mocked(networking.getGuardrailInfo).mockResolvedValue( + guardrail({ guardrailIdentifier: "gr-abc", mode: "logging_only", logging_only_scope: "output" }), + ); + const user = userEvent.setup({ delay: null }); + renderView(); + await openEditor(user); + await saveChanges(user); + + await waitFor(() => expect(networking.updateGuardrailCall).toHaveBeenCalledTimes(1)); + expect(lastPayload()).toEqual({ litellm_params: { logging_only_scope: null } }); + }); + it("parses the guardrail information textarea into an object", async () => { const user = userEvent.setup({ delay: null }); renderView(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx index 73d9fb8123e..57748d3fdb8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx @@ -38,6 +38,7 @@ import { getLoggingOnlyScopeUpdate, getGuardrailLogoAndName, guardrail_provider_map, + choiceToLoggingOnlyScope, loggingOnlyScopeToChoice, skipSystemMessageToChoice, skipToolMessageToChoice, @@ -224,27 +225,27 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, // binds are seeded: an unbound key would otherwise be submitted as if the user had set it. useEffect(() => { if (!guardrailData) return; + const litellmParams = guardrailData.litellm_params; form.setValue("guardrail_name", guardrailData.guardrail_name); - form.setValue("default_on", guardrailData.litellm_params?.default_on); - const storedLoggingOnlyScope = guardrailData.litellm_params?.logging_only_scope; - form.setValue("logging_only_scope_choice", loggingOnlyScopeToChoice(storedLoggingOnlyScope)); - form.setValue( - "skip_system_message_choice", - skipSystemMessageToChoice(guardrailData.litellm_params?.skip_system_message_in_guardrail), - ); - form.setValue( - "skip_tool_message_choice", - skipToolMessageToChoice(guardrailData.litellm_params?.skip_tool_message_in_guardrail), - ); - form.setValue( - "guardrail_info", - guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", - ); - if (guardrailData.litellm_params?.optional_params) { - form.setValue("optional_params", guardrailData.litellm_params.optional_params); + form.setValue("default_on", litellmParams?.default_on); + form.setValue("logging_only_scope_choice", loggingOnlyScopeToChoice(litellmParams?.logging_only_scope)); + const skipSystemMessageChoice = skipSystemMessageToChoice(litellmParams?.skip_system_message_in_guardrail); + form.setValue("skip_system_message_choice", skipSystemMessageChoice); + form.setValue("skip_tool_message_choice", skipToolMessageToChoice(litellmParams?.skip_tool_message_in_guardrail)); + const guardrailInfo = guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : ""; + form.setValue("guardrail_info", guardrailInfo); + if (litellmParams?.optional_params) { + form.setValue("optional_params", litellmParams.optional_params); } }, [guardrailData, guardrailProviderSpecificParams, form]); + useEffect(() => { + const scope = choiceToLoggingOnlyScope(form.getValues("logging_only_scope_choice")); + if (!directionalScopeSupported && scope !== null && scope !== "both") { + form.setValue("logging_only_scope_choice", "default"); + } + }, [directionalScopeSupported, form, guardrailData]); + const resetToolPermissionEditor = useCallback(() => { if (guardrailData?.litellm_params?.guardrail === "tool_permission") { setToolPermissionConfig({ From d3bb9a6a4ac13b45d6486a565859a398bb6737ac Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 10:54:25 +0000 Subject: [PATCH 13/20] fix(guardrails): normalize logging-only scope and sanitize warning logs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/guardrails/guardrail_registry.py | 2 +- ...d_guardrail_form.characterization.test.tsx | 30 +-------------- .../_components/add_guardrail_form.tsx | 6 +-- .../guardrails/_components/guardrail_info.tsx | 37 +++++++++++-------- .../guardrail_info_helpers.test.tsx | 10 +++++ .../_components/guardrail_info_helpers.tsx | 6 +++ 6 files changed, 43 insertions(+), 48 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index ea06f4993b1..404468a387d 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -472,7 +472,7 @@ def _configure_callback_scoping( raise ValueError(logging_only_scope_error) verbose_proxy_logger.error( "%s Ignoring logging_only_scope; the guardrail keeps its configured mode.", - logging_only_scope_error, + logging_only_scope_error.replace("\r", "").replace("\n", ""), ) custom_guardrail_callback.logging_only_scope = None else: diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx index 040d04b7f48..230be13ef33 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { fireEvent, screen, waitFor } from "@testing-library/react"; +import { screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { chooseSelectOption, renderWithProviders } from "@/../tests/test-utils"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -151,34 +151,6 @@ describe("AddGuardrailForm create payload characterization", () => { expect(screen.getByRole("option", { name: "Both (request and response)" })).toBeInTheDocument(); }); - it("clears a selected directional scope when switching to an unsupported provider", async () => { - vi.mocked(networking.getGuardrailUISettings).mockResolvedValue({ - ...uiSettings, - supported_modes: ["pre_call", "logging_only"], - providers_without_directional_logging_only_scope: ["xecguard"], - }); - vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({ - ...providerParams, - xecguard: { ui_friendly_name: "XecGuard" }, - }); - const user = userEvent.setup({ delay: null }); - renderForm(); - - fireEvent.change(await screen.findByLabelText("Guardrail Name"), { target: { value: "switch-scope" } }); - await pickProvider(user, "Bedrock Guardrail"); - await user.click(screen.getByLabelText("Mode")); - await user.click((await screen.findAllByText("logging_only")).at(-1) as HTMLElement); - await chooseSelectOption(user, await screen.findByLabelText("Logging only scope"), "Output only (response)"); - expect(screen.getByLabelText("Logging only scope")).toHaveTextContent("Output only (response)"); - await pickProvider(user, "XecGuard"); - await user.click(screen.getByRole("button", { name: "Next" })); - await user.click(await screen.findByRole("button", { name: "Create Guardrail" })); - - await waitFor(() => expect(networking.createGuardrailCall).toHaveBeenCalledTimes(1)); - expect(payload()?.litellm_params.mode).toContain("logging_only"); - expect(payload()?.litellm_params).not.toHaveProperty("logging_only_scope"); - }); - it("hides logging-only scope and omits it from a pre-call payload", async () => { const user = userEvent.setup({ delay: null }); renderForm(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx index d99a31fedf0..0979a1e27a6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx @@ -18,6 +18,7 @@ import { getSupportedModesForProvider, guardrail_provider_map, modeIncludesLoggingOnly, + normalizeLoggingOnlyScopeChoice, populateGuardrailProviderMap, populateGuardrailProviders, shouldRenderContentFilterConfigSettings, @@ -247,9 +248,8 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a useEffect(() => { const scopeChoice = form.getValues("logging_only_scope_choice"); - if (!directionalScopeSupported && (scopeChoice === "input" || scopeChoice === "output")) { - form.setValue("logging_only_scope_choice", "default"); - } + const normalizedScopeChoice = normalizeLoggingOnlyScopeChoice(scopeChoice, directionalScopeSupported); + if (normalizedScopeChoice !== scopeChoice) form.setValue("logging_only_scope_choice", normalizedScopeChoice); }, [directionalScopeSupported, form]); // Fetch guardrail UI settings + provider params on mount / accessToken change diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx index 57748d3fdb8..aa8515874b0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx @@ -38,8 +38,8 @@ import { getLoggingOnlyScopeUpdate, getGuardrailLogoAndName, guardrail_provider_map, - choiceToLoggingOnlyScope, loggingOnlyScopeToChoice, + normalizeLoggingOnlyScopeChoice, skipSystemMessageToChoice, skipToolMessageToChoice, supportsDirectionalLoggingOnlyScope, @@ -225,25 +225,31 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, // binds are seeded: an unbound key would otherwise be submitted as if the user had set it. useEffect(() => { if (!guardrailData) return; - const litellmParams = guardrailData.litellm_params; form.setValue("guardrail_name", guardrailData.guardrail_name); - form.setValue("default_on", litellmParams?.default_on); - form.setValue("logging_only_scope_choice", loggingOnlyScopeToChoice(litellmParams?.logging_only_scope)); - const skipSystemMessageChoice = skipSystemMessageToChoice(litellmParams?.skip_system_message_in_guardrail); - form.setValue("skip_system_message_choice", skipSystemMessageChoice); - form.setValue("skip_tool_message_choice", skipToolMessageToChoice(litellmParams?.skip_tool_message_in_guardrail)); - const guardrailInfo = guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : ""; - form.setValue("guardrail_info", guardrailInfo); - if (litellmParams?.optional_params) { - form.setValue("optional_params", litellmParams.optional_params); + form.setValue("default_on", guardrailData.litellm_params?.default_on); + const storedLoggingOnlyScope = guardrailData.litellm_params?.logging_only_scope; + form.setValue("logging_only_scope_choice", loggingOnlyScopeToChoice(storedLoggingOnlyScope)); + form.setValue( + "skip_system_message_choice", + skipSystemMessageToChoice(guardrailData.litellm_params?.skip_system_message_in_guardrail), + ); + form.setValue( + "skip_tool_message_choice", + skipToolMessageToChoice(guardrailData.litellm_params?.skip_tool_message_in_guardrail), + ); + form.setValue( + "guardrail_info", + guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", + ); + if (guardrailData.litellm_params?.optional_params) { + form.setValue("optional_params", guardrailData.litellm_params.optional_params); } }, [guardrailData, guardrailProviderSpecificParams, form]); useEffect(() => { - const scope = choiceToLoggingOnlyScope(form.getValues("logging_only_scope_choice")); - if (!directionalScopeSupported && scope !== null && scope !== "both") { - form.setValue("logging_only_scope_choice", "default"); - } + const scopeChoice = form.getValues("logging_only_scope_choice"); + const normalizedScopeChoice = normalizeLoggingOnlyScopeChoice(scopeChoice, directionalScopeSupported); + if (normalizedScopeChoice !== scopeChoice) form.setValue("logging_only_scope_choice", normalizedScopeChoice); }, [directionalScopeSupported, form, guardrailData]); const resetToolPermissionEditor = useCallback(() => { @@ -519,6 +525,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, const isConfigGuardrail = guardrailData.guardrail_definition_location === "config"; + /* eslint-disable max-lines -- keep edit-form scope normalization with its owning view */ return (
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx index de1be697221..96ade4b8c1b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx @@ -21,6 +21,7 @@ import { getLoggingOnlyScopeOptions, formatLoggingOnlyScope, modeIncludesLoggingOnly, + normalizeLoggingOnlyScopeChoice, supportsDirectionalLoggingOnlyScope, } from "./guardrail_info_helpers"; @@ -248,6 +249,15 @@ describe("guardrail_info_helpers", () => { }); describe("logging-only scope helpers", () => { + it("normalizes directional choices only when the provider does not support them", () => { + expect(normalizeLoggingOnlyScopeChoice("input", false)).toBe("default"); + expect(normalizeLoggingOnlyScopeChoice("output", false)).toBe("default"); + expect(normalizeLoggingOnlyScopeChoice("both", false)).toBe("both"); + expect(normalizeLoggingOnlyScopeChoice("default", false)).toBe("default"); + expect(normalizeLoggingOnlyScopeChoice("input", true)).toBe("input"); + expect(normalizeLoggingOnlyScopeChoice("output", true)).toBe("output"); + }); + it("maps API scope values to choices and back", () => { expect(loggingOnlyScopeToChoice("input")).toBe("input"); expect(loggingOnlyScopeToChoice("output")).toBe("output"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx index da28e0962a8..30162a2b942 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx @@ -118,6 +118,12 @@ export type LoggingOnlyScope = "input" | "output" | "both"; export type LoggingOnlyScopeChoice = "default" | LoggingOnlyScope; export type LoggingOnlyScopeOption = { label: string; value: LoggingOnlyScopeChoice }; +export const normalizeLoggingOnlyScopeChoice = ( + choice: LoggingOnlyScopeChoice, + directionalScopeSupported: boolean, +): LoggingOnlyScopeChoice => + directionalScopeSupported || choice === "default" || choice === "both" ? choice : "default"; + const LOGGING_ONLY_SCOPE_OPTIONS: LoggingOnlyScopeOption[] = [ { label: "Default (request and response)", value: "default" }, { label: "Input only (request)", value: "input" }, From ef1a1f42ec7aef4b39ade6fa45bf0843228b61b2 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 29 Sep 2026 10:57:36 +0000 Subject: [PATCH 14/20] fix(ui): normalize scope in shared guardrail field Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrails/_components/GuardrailFormField.tsx | 12 +++++++++++- .../guardrails/_components/add_guardrail_form.tsx | 7 ------- .../guardrails/_components/guardrail_info.tsx | 8 -------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx index 3b76c9e8016..100d23ad5bb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailFormField.tsx @@ -1,7 +1,7 @@ "use client"; import { CircleHelp } from "lucide-react"; -import React, { useId } from "react"; +import React, { useEffect, useId } from "react"; import { useController, type Control, type ControllerRenderProps, type RegisterOptions } from "react-hook-form"; import { Field, FieldDescription, FieldError, FieldLabel } from "@/components/ui/field"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; @@ -9,6 +9,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip import { getLoggingOnlyScopeOptions, modeIncludesLoggingOnly, + normalizeLoggingOnlyScopeChoice, type LoggingOnlyScopeChoice, } from "./guardrail_info_helpers"; @@ -44,6 +45,9 @@ export const asText = (value: unknown): string => { return ""; }; +const isLoggingOnlyScopeChoice = (value: unknown): value is LoggingOnlyScopeChoice => + value === "default" || value === "input" || value === "output" || value === "both"; + export const asStringArray = (value: unknown): string[] => { if (Array.isArray(value)) return value.filter((entry): entry is string => typeof entry === "string"); if (typeof value === "string" && value !== "") return [value]; @@ -137,6 +141,12 @@ export const LoggingOnlyScopeSelect: React.FC<{ const { id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy } = control; const items = getLoggingOnlyScopeOptions(directionalScopeSupported); + useEffect(() => { + const currentChoice = isLoggingOnlyScopeChoice(value) ? value : "default"; + const choice = normalizeLoggingOnlyScopeChoice(currentChoice, directionalScopeSupported); + if (choice !== value) onChange(choice); + }, [value, directionalScopeSupported, onChange]); + return (