fix(guardrails): stop logging_only llm_as_a_judge from judging its own judge calls

Judge sub-calls now carry the internal_call_origin metadata stamp and the
guardrail skips any logged call bearing it, so a logging_only judge no longer
recurses into an unbounded chain of judge requests

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/proxy/guardrails/guardrail_hooks/llm_as_a_judge/__init__.py

@@ -1,15 +1,17 @@
 """LLM-as-a-Judge guardrail: uses an LLM to score requests or responses against weighted criteria."""
 
-from collections.abc import Callable, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from datetime import datetime
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeVar
 
 from fastapi import HTTPException
+from pydantic import BaseModel, ConfigDict, ValidationError
 from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.litellm_core_utils.llm_judge import (
     default_router_provider,
@@ -18,7 +20,7 @@ from litellm.litellm_core_utils.llm_judge import (
     parse_json_verdict,
 )
 from litellm.types.guardrails import GuardrailEventHooks, Mode, SupportedGuardrailIntegrations
-from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailStatus
+from litellm.types.utils import LLM_AS_A_JUDGE_GUARDRAIL_CALL_ORIGIN, GenericGuardrailAPIInputs, GuardrailStatus
 
 if TYPE_CHECKING:
     from litellm import Router
@@ -57,6 +59,25 @@ _JUDGE_SUBJECT_LABELS: Final[MappingProxyType[JudgeInputType, str]] = MappingPro
 
 _VALID_ON_FAILURE: Final = frozenset({"block", "log"})
 
+_JUDGE_CALL_METADATA: Final = MappingProxyType(
+    {INTERNAL_CALL_ORIGIN_METADATA_KEY: LLM_AS_A_JUDGE_GUARDRAIL_CALL_ORIGIN}
+)
+
+
+class _LoggedCallParams(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    metadata: Mapping[str, object] | None = None
+
+
+def _is_judge_call(data: Mapping[str, object]) -> bool:
+    try:
+        params: Final = _LoggedCallParams.model_validate(data.get("litellm_params") or {})
+    except ValidationError:
+        return False
+    return (params.metadata or {}).get(INTERNAL_CALL_ORIGIN_METADATA_KEY) == LLM_AS_A_JUDGE_GUARDRAIL_CALL_ORIGIN
+
+
 _default_router_provider: Final = default_router_provider
 _parse_judge_verdict: Final = parse_json_verdict
 _extract_text_from_content: Final = extract_text_from_content
@@ -169,6 +190,11 @@ class LLMAsAJudgeGuardrail(CustomGuardrail):
     def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]:
         return [GuardrailEventHooks.pre_call, GuardrailEventHooks.during_call, GuardrailEventHooks.post_call]
 
+    def should_run_guardrail(self, data: Mapping[str, object], event_type: GuardrailEventHooks) -> bool:
+        if _is_judge_call(data):
+            return False
+        return super().should_run_guardrail(data, event_type)
+
     async def _run_judge(
         self,
         messages: Sequence[JudgeMessage],
@@ -188,6 +214,7 @@ class LLMAsAJudgeGuardrail(CustomGuardrail):
             judge_messages,
             response_format={"type": "json_object"},
             temperature=0,
+            metadata=dict(_JUDGE_CALL_METADATA),
         )
         raw: Final = response.choices[0].message.content or "{}"
         return _parse_judge_verdict(raw)

litellm/types/utils.py

@@ -2957,6 +2957,7 @@ InternalCallOrigin = Literal[
     "autorouter_classifier",
     "shadow_eval_router",
     "shadow_eval_judge",
+    "llm_as_a_judge_guardrail",
     "background_response_cost_poll",
 ]
 """Which internal litellm feature originated a billed sub-call, so a spend log row
@@ -2965,6 +2966,7 @@ records that it is not traffic the caller sent."""
 AUTOROUTER_CLASSIFIER_CALL_ORIGIN: Final[InternalCallOrigin] = "autorouter_classifier"
 SHADOW_EVAL_ROUTER_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_router"
 SHADOW_EVAL_JUDGE_CALL_ORIGIN: Final[InternalCallOrigin] = "shadow_eval_judge"
+LLM_AS_A_JUDGE_GUARDRAIL_CALL_ORIGIN: Final[InternalCallOrigin] = "llm_as_a_judge_guardrail"
 BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN: Final[InternalCallOrigin] = "background_response_cost_poll"
 
 

tests/test_litellm/proxy/guardrails/test_llm_as_a_judge.py

@@ -306,6 +306,22 @@ async def test_apply_guardrail_logging_only_labels_both_sides_logging_only(input
     assert request_data["metadata"]["standard_logging_guardrail_information"][0]["guardrail_mode"] == "logging_only"
 
 
+@pytest.mark.asyncio
+async def test_logging_only_judge_does_not_judge_its_own_judge_call():
+    router: Final = _judge_router(90.0)
+    guardrail: Final = _make_guardrail(event_hook=GuardrailEventHooks.logging_only, router_provider=lambda: router)
+    client_call: Final[dict[str, object]] = {"litellm_params": {"metadata": {"user_api_key": "hashed"}}}
+
+    assert guardrail.should_run_guardrail(client_call, GuardrailEventHooks.logging_only) is True
+    await guardrail.apply_guardrail({"texts": ["hi"]}, {"messages": [{"role": "user", "content": "hi"}]}, "request")
+
+    judge_call: Final[dict[str, object]] = {
+        "litellm_params": {"metadata": router.acompletion.call_args.kwargs["metadata"]}
+    }
+    assert guardrail.should_run_guardrail(judge_call, GuardrailEventHooks.logging_only) is False
+    assert guardrail.should_run_guardrail(client_call, GuardrailEventHooks.logging_only) is True
+
+
 @pytest.mark.asyncio
 async def test_apply_guardrail_response_prompt_unchanged():
     router: Final = _judge_router(90.0)