diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 6befbab72e9..2d66a280663 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -601,12 +601,6 @@ class CustomGuardrail(CustomLogger): event_hook: GuardrailEventHooks | list[GuardrailEventHooks] | Mode | None, supported_event_hooks: list[GuardrailEventHooks], ) -> None: - allowed_hooks: Final = frozenset(supported_event_hooks) | ( - frozenset((GuardrailEventHooks.logging_only,)) - if self.uses_apply_guardrail_interface() and not self.use_native_lifecycle_hooks - else frozenset() - ) - def _validate_event_hook_list_is_in_supported_event_hooks( event_hook: list[GuardrailEventHooks] | list[str], supported_event_hooks: list[GuardrailEventHooks], @@ -614,7 +608,7 @@ class CustomGuardrail(CustomLogger): for hook in event_hook: if isinstance(hook, str): hook = GuardrailEventHooks(hook) - if hook not in allowed_hooks: + if hook not in supported_event_hooks: raise ValueError(f"Event hook {hook} is not in the supported event hooks {supported_event_hooks}") if event_hook is None: @@ -635,7 +629,7 @@ class CustomGuardrail(CustomLogger): default_list = event_hook.default if isinstance(event_hook.default, list) else [event_hook.default] _validate_event_hook_list_is_in_supported_event_hooks(default_list, supported_event_hooks) elif isinstance(event_hook, GuardrailEventHooks): - if event_hook not in allowed_hooks: + if event_hook not in supported_event_hooks: raise ValueError(f"Event hook {event_hook} is not in the supported event hooks {supported_event_hooks}") @staticmethod diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py index baf3c6ae41f..885bd1d4d72 100644 --- a/tests/test_litellm/integrations/test_custom_guardrail.py +++ b/tests/test_litellm/integrations/test_custom_guardrail.py @@ -1,5 +1,5 @@ import asyncio -from typing import TYPE_CHECKING, Final, Literal, Optional +from typing import TYPE_CHECKING, Literal, Optional from unittest.mock import AsyncMock import pytest @@ -2383,30 +2383,6 @@ class TestLoggingOnlyApplyGuardrail: apply_guardrail must still run against the logged request and response and record guardrail_information, instead of inheriting the CustomLogger no-op.""" - @pytest.mark.asyncio - async def test_content_filter_accepts_logging_only_and_records_detection(self): - from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ( - ContentFilterGuardrail, - ) - from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks - - guardrail: Final = ContentFilterGuardrail( - guardrail_name="content-review", - event_hook=GuardrailEventHooks.logging_only, - default_on=True, - blocked_words=[BlockedWord(keyword="hello", action=ContentFilterAction.BLOCK)], - ) - kwargs, response = _logged_call([{"role": "user", "content": "hello there"}]) - - out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value) - - assert out_response is response - assert out_kwargs["messages"] == kwargs["messages"] - assert ( - out_kwargs["standard_logging_object"]["guardrail_information"][0]["guardrail_status"] - == "guardrail_intervened" - ) - @pytest.mark.asyncio async def test_runs_apply_guardrail_observe_only_and_records_verdict(self): guardrail = _ApplyOnlyObserver() diff --git a/tests/test_litellm_rust/test_integrations.py b/tests/test_litellm_rust/test_integrations.py index f7632a7bdb1..6878591e6e8 100644 --- a/tests/test_litellm_rust/test_integrations.py +++ b/tests/test_litellm_rust/test_integrations.py @@ -40,6 +40,12 @@ pytestmark = pytest.mark.requires_rust_extension FAILURE_RESPONSE: Final = ResponseSpec(body={"message": "provider unavailable"}, status=500) +class LoggingOnlyContentFilter(ContentFilterGuardrail): + @classmethod + def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: + return [*super().get_supported_event_hooks(), GuardrailEventHooks.logging_only] + + @pytest.mark.asyncio @pytest.mark.parametrize("route", ASYNC_ROUTES, ids=route_id) async def test_generic_api_logger_exports_success_over_http(route: Route, provider: RecordingServer) -> None: @@ -122,6 +128,7 @@ async def test_generic_guardrail_logging_only_verdict_is_exported_over_http( guardrail: Final = GenericGuardrailAPI( api_base=provider.base_url, guardrail_name="http-review", + supported_event_hooks=[*GenericGuardrailAPI.get_supported_event_hooks(), GuardrailEventHooks.logging_only], event_hook=GuardrailEventHooks.logging_only, default_on=True, ) @@ -158,7 +165,7 @@ async def test_generic_guardrail_logging_only_verdict_is_exported_over_http( async def test_content_filter_logging_only_detects_real_content_without_changing_response( route: Route, provider: RecordingServer, action: ContentFilterAction ) -> None: - guardrail: Final = ContentFilterGuardrail( + guardrail: Final = LoggingOnlyContentFilter( guardrail_name="content-review", event_hook=GuardrailEventHooks.logging_only, default_on=True,