From 28bd00a004fca77f1608f8d42f07e136a0b14b2e Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 14 Sep 2026 21:53:20 +0000 Subject: [PATCH] fix(guardrails): label llm_as_a_judge logging_only verdicts with their mode Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../guardrail_hooks/llm_as_a_judge/__init__.py | 2 ++ .../proxy/guardrails/test_llm_as_a_judge.py | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py index 269296943cb..9f0c7476b4c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py @@ -277,6 +277,8 @@ class LLMAsAJudgeGuardrail(CustomGuardrail): ) def _event_type_for(self, input_type: JudgeInputType) -> GuardrailEventHooks: + if self._event_hook_is_event_type(GuardrailEventHooks.logging_only): + return GuardrailEventHooks.logging_only if input_type == "response": return GuardrailEventHooks.post_call if self._event_hook_is_event_type(GuardrailEventHooks.pre_call): diff --git a/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py b/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py index 11e9e55cea4..fc877b323e0 100644 --- a/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py +++ b/tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py @@ -285,6 +285,24 @@ async def test_apply_guardrail_request_multi_turn_keeps_roles_and_focuses_latest ) in judge_messages[1]["content"] +@pytest.mark.asyncio +@pytest.mark.parametrize("input_type", ["request", "response"]) +async def test_apply_guardrail_logging_only_labels_both_sides_logging_only(input_type: str): + router: Final = _judge_router(50.0) + guardrail: Final = _make_guardrail( + on_failure="log", + event_hook=GuardrailEventHooks.logging_only, + router_provider=lambda: router, + ) + request_data: Final[dict[str, object]] = {"messages": [{"role": "user", "content": "hi"}], "metadata": {}} + + assert guardrail.should_run_guardrail(request_data, GuardrailEventHooks.pre_call) is False + assert guardrail.should_run_guardrail(request_data, GuardrailEventHooks.post_call) is False + await guardrail.apply_guardrail({"texts": ["hi"]}, request_data, input_type) + + assert request_data["metadata"]["standard_logging_guardrail_information"][0]["guardrail_mode"] == "logging_only" + + @pytest.mark.asyncio async def test_apply_guardrail_response_prompt_unchanged(): router: Final = _judge_router(90.0)