Merge cb72e989f6 into b4a8bc715f

litellm/litellm_core_utils/litellm_logging.py

@@ -1873,8 +1873,9 @@ class Logging(LiteLLMLoggingBaseClass):
                     verbose_logger.exception(
                         "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e
                     )
+            post_call_details: Final = self._logging_details_copy(self.model_call_details)
             redact_message_input_output_from_logging(
-                model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
+                model_call_details=post_call_details,
                 result=original_response,
             )
             # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made
@@ -1888,9 +1889,9 @@ class Logging(LiteLLMLoggingBaseClass):
                     if callback == "sentry" and add_breadcrumb:
                         verbose_logger.debug("reaches sentry breadcrumbing")
                         try:
-                            details_to_log = copy.deepcopy(self.model_call_details)
+                            details_to_log = copy.deepcopy(post_call_details)
                         except Exception:
-                            details_to_log = self.model_call_details
+                            details_to_log = post_call_details
                         if litellm.turn_off_message_logging:
                             # make a copy of the _model_Call_details and log it
                             details_to_log.pop("messages", None)
@@ -1904,7 +1905,7 @@ class Logging(LiteLLMLoggingBaseClass):
                         )
                     elif isinstance(callback, CustomLogger):  # custom logger class
                         callback.log_post_api_call(
-                            kwargs=self.model_call_details,
+                            kwargs=post_call_details,
                             response_obj=None,
                             start_time=self.start_time,
                             end_time=None,
@@ -3111,9 +3112,7 @@ class Logging(LiteLLMLoggingBaseClass):
         if not build_logging_payload:
             return None
 
-        standard_logging_payload: Final = self._build_standard_logging_payload(
-            logging_result, start_time, end_time, isolate_call_details=not publish_logging_payload
-        )
+        standard_logging_payload: Final = self._build_standard_logging_payload(logging_result, start_time, end_time)
         if not publish_logging_payload:
             return standard_logging_payload
         self.model_call_details["standard_logging_object"] = standard_logging_payload
@@ -3134,17 +3133,26 @@ class Logging(LiteLLMLoggingBaseClass):
 
         await invalidate_baseline_cache(self, reason, completed=completed)
 
+    def _logging_details_copy(
+        self, details: Mapping[str, object]
+    ) -> dict[str, object]:  # mutable-ok: legacy redaction mutates its isolated payload
+        streaming_keys: Final = ("complete_streaming_response", "async_complete_streaming_response")
+        redact: Final = should_redact_message_logging(self.model_call_details)
+        return {
+            **details,
+            **{key: copy.deepcopy(details[key]) for key in streaming_keys if redact and key in details},
+        }
+
     def _build_standard_logging_payload(
         self,
         init_response_obj: object,
         start_time: dt_object,
         end_time: dt_object,
-        isolate_call_details: bool = False,
     ) -> StandardLoggingPayload | None:
         """Build StandardLoggingPayload and accumulate its construction time."""
         _start: Final = time.time()
         payload: Final = get_standard_logging_object_payload(
-            kwargs=dict(self.model_call_details) if isolate_call_details else self.model_call_details,
+            kwargs=self._logging_details_copy(self.model_call_details),
             init_response_obj=init_response_obj,
             start_time=start_time,
             end_time=end_time,
@@ -3696,7 +3704,6 @@ class Logging(LiteLLMLoggingBaseClass):
                     self.model_call_details["complete_streaming_response"],
                     start_time,
                     end_time,
-                    isolate_call_details=True,
                 )
                 if for_async_request and complete_streaming_response is not None
                 else request_logging_payload
@@ -3729,35 +3736,47 @@ class Logging(LiteLLMLoggingBaseClass):
                 else callbacks
             )
 
-            ## REDACT MESSAGES ##
-            result = _REDACTED_RESULT.validate_python(
-                redact_message_input_output_from_logging(
-                    model_call_details=call_details,
-                    result=result,
-                )
-            )
             ## LOGGING HOOK ##
             hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
+            if should_redact_message_logging(call_details) and any(
+                isinstance(callback, CustomGuardrail) for callback in hook_callbacks
+            ):
+                call_details = {  # rebind-ok: logging hooks consume and replace this isolated legacy payload
+                    **self._logging_details_copy(call_details),
+                    "messages": copy.deepcopy(call_details.get("messages")),
+                }
+                result = copy.deepcopy(result)  # rebind-ok: preserve the client response before mutable logging hooks
             for callback in hook_callbacks:
-                if isinstance(callback, CustomGuardrail):
-                    from litellm.types.guardrails import GuardrailEventHooks
+                try:
+                    if isinstance(callback, CustomGuardrail):
+                        from litellm.types.guardrails import GuardrailEventHooks
 
-                    if (
-                        callback.should_run_guardrail(
-                            data=call_details,
-                            event_type=GuardrailEventHooks.logging_only,
+                        if (
+                            callback.should_run_guardrail(
+                                data=call_details,
+                                event_type=GuardrailEventHooks.logging_only,
+                            )
+                            is not True
+                        ):
+                            continue
+
+                        self.shadow_eval_request_snapshot = None
+                        call_details, result = callback.logging_hook(  # pyright: ignore[reportUnknownVariableType]  # logging_hook is typed with a bare dict
+                            kwargs=call_details,
+                            result=result,
+                            call_type=self.call_type,
                         )
-                        is not True
-                    ):
-                        continue
+                except Exception:  # noqa: BLE001  # a failed guardrail must not skip redaction or later logging callbacks
+                    verbose_logger.exception("LiteLLM.LoggingError: Exception in guardrail logging_hook")
+                    self._handle_callback_failure(callback=callback)
+                if not for_async_request:
+                    self.model_call_details = call_details
 
-                    self.shadow_eval_request_snapshot = None
-                    call_details, result = callback.logging_hook(  # pyright: ignore[reportUnknownVariableType]  # logging_hook is typed with a bare dict
-                        kwargs=call_details,
-                        result=result,
-                        call_type=self.call_type,
-                    )
-                elif isinstance(callback, CustomLogger):
+            result = _REDACTED_RESULT.validate_python(
+                redact_message_input_output_from_logging(model_call_details=call_details, result=result)
+            )
+            for callback in hook_callbacks:
+                if isinstance(callback, CustomLogger) and not isinstance(callback, CustomGuardrail):
                     call_details, result = callback.logging_hook(  # pyright: ignore[reportUnknownVariableType]  # logging_hook is typed with a bare dict
                         kwargs=call_details,
                         result=result,
@@ -4320,16 +4339,17 @@ class Logging(LiteLLMLoggingBaseClass):
             self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks
         )
 
-        result = _REDACTED_RESULT.validate_python(
-            redact_message_input_output_from_logging(
-                model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
-                result=result,
-            )
-        )
-
         ## LOGGING HOOK ##
 
         hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
+        if should_redact_message_logging(self.model_call_details) and any(
+            isinstance(callback, CustomGuardrail) for callback in hook_callbacks
+        ):
+            self.model_call_details = {
+                **self._logging_details_copy(self.model_call_details),
+                "messages": copy.deepcopy(self.model_call_details.get("messages")),
+            }
+            result = copy.deepcopy(result)  # rebind-ok: preserve the client response before mutable logging hooks
         for callback in hook_callbacks:
             try:
                 if isinstance(callback, CustomGuardrail):
@@ -4350,7 +4370,16 @@ class Logging(LiteLLMLoggingBaseClass):
                         result=result,
                         call_type=self.call_type,
                     )
-                elif isinstance(callback, CustomLogger):
+            except Exception:  # noqa: BLE001  # a failed guardrail must not skip redaction or later logging callbacks
+                verbose_logger.exception("LiteLLM.LoggingError: Exception in guardrail async_logging_hook")
+                self._handle_callback_failure(callback=callback)
+
+        result = _REDACTED_RESULT.validate_python(
+            redact_message_input_output_from_logging(model_call_details=self.model_call_details, result=result)
+        )
+        for callback in hook_callbacks:
+            try:
+                if isinstance(callback, CustomLogger) and not isinstance(callback, CustomGuardrail):
                     result = redact_message_input_output_from_custom_logger(
                         result=result, litellm_logging_obj=self, custom_logger=callback
                     )

litellm/litellm_core_utils/redact_messages.py

@@ -22,6 +22,7 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.classifier_logging import without_classifier_audit
 from litellm.litellm_core_utils.core_helpers import (
     get_metadata_variable_name_from_kwargs,
+    redact_nested_match_and_regex_keys,  # pyright: ignore[reportUnknownVariableType]  # shared legacy helper uses bare dicts
 )
 from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY
 from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items
@@ -31,7 +32,7 @@ from litellm.llms.vertex_ai.common_utils import (
 )
 from litellm.secret_managers.main import str_to_bool
 from litellm.types.router import BaselineRouteStamp
-from litellm.types.utils import StandardCallbackDynamicParams
+from litellm.types.utils import PROMPT_CARRYING_GUARDRAIL_FIELDS, StandardCallbackDynamicParams
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import (
@@ -432,6 +433,10 @@ def _redact_standard_logging_object(
 
     if standard_logging_object.get("messages") is not None:
         standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}]
+    if "guardrail_information" in standard_logging_object:
+        standard_logging_object["guardrail_information"] = _redacted_guardrail_information(
+            standard_logging_object["guardrail_information"]
+        )
     model_parameters: Final = _as_mapping(standard_logging_object.get("model_parameters"))
     if model_parameters is not None and redact_instructions:
         standard_logging_object["model_parameters"] = _with_redacted_instructions(model_parameters)
@@ -492,11 +497,33 @@ def _redact_model_response_dict_choices(choices: Iterable[object], redacted_str:
             _redact_choice_content(choice)
 
 
-def _redacted_baseline_metadata(metadata: Mapping[str, object]) -> Mapping[str, object]:
+def _redacted_guardrail_information(value: object) -> object:
+    if not isinstance(value, (dict, list)):
+        return value
+    return redact_nested_match_and_regex_keys(value, keys=PROMPT_CARRYING_GUARDRAIL_FIELDS)  # pyright: ignore[reportUnknownVariableType]  # opaque result of the legacy redactor
+
+
+def _redacted_logging_metadata(metadata: Mapping[str, object]) -> Mapping[str, object]:
     route: Final = metadata.get("_autorouter_baseline_route")
-    if not isinstance(route, BaselineRouteStamp):
+    if not isinstance(route, BaselineRouteStamp) and "standard_logging_guardrail_information" not in metadata:
         return metadata
-    return {**metadata, "_autorouter_baseline_route": replace(route, request_parameters=None)}
+    return {
+        **metadata,
+        **(
+            {"_autorouter_baseline_route": replace(route, request_parameters=None)}
+            if isinstance(route, BaselineRouteStamp)
+            else {}
+        ),
+        **(
+            {
+                "standard_logging_guardrail_information": _redacted_guardrail_information(
+                    metadata["standard_logging_guardrail_information"]
+                )
+            }
+            if "standard_logging_guardrail_information" in metadata
+            else {}
+        ),
+    }
 
 
 def _redacted_proxy_server_request(request: Mapping[str, object], *, redact_body: bool) -> Mapping[str, object]:
@@ -533,7 +560,7 @@ def redacted_litellm_params(params: Mapping[str, object], *, redact_request_body
     return {
         **(_with_redacted_instructions(params) if redact_request_body else params),
         **{
-            key: _redacted_baseline_metadata(value)
+            key: _redacted_logging_metadata(value)
             for key, value in params.items()
             if key in ("metadata", "litellm_metadata") and isinstance(value, Mapping)
         },

tests/unit/litellm_core_utils/test_litellm_logging.py

@@ -3632,6 +3632,230 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g
     assert "snapshot-only" not in json.dumps(logging_obj.model_call_details, default=str)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("asynchronous", [False, True])
+@pytest.mark.parametrize("redact", [False, True])
+@pytest.mark.parametrize("stream", [False, True])
+async def test_logging_guardrail_screens_original_content_while_loggers_honor_redaction(
+    monkeypatch: pytest.MonkeyPatch, asynchronous: bool, redact: bool, stream: bool
+) -> None:
+    from queue import SimpleQueue
+
+    from litellm.integrations.custom_guardrail import CustomGuardrail
+    from litellm.types.guardrails import GuardrailEventHooks
+
+    screened: Final[SimpleQueue[str]] = SimpleQueue()
+    logged: Final[SimpleQueue[str]] = SimpleQueue()
+
+    class ScreeningGuardrail(CustomGuardrail):
+        def logging_hook(
+            self, kwargs: dict[str, object], result: object, call_type: str
+        ) -> tuple[dict[str, object], object]:
+            screened.put(json.dumps([kwargs["messages"], str(result)]))
+            if redact and isinstance(result, ModelResponse):
+                result.choices[0].message.content = "masked-by-guardrail"
+            return kwargs, result
+
+        async def async_logging_hook(
+            self, kwargs: dict[str, object], result: object, call_type: str
+        ) -> tuple[dict[str, object], object]:
+            return self.logging_hook(kwargs, result, call_type)
+
+    class RecordingLogger(CustomLogger):
+        def logging_hook(
+            self, kwargs: dict[str, object], result: object, call_type: str
+        ) -> tuple[dict[str, object], object]:
+            logged.put(json.dumps([kwargs["messages"], str(result)]))
+            return kwargs, result
+
+        async def async_logging_hook(
+            self, kwargs: dict[str, object], result: object, call_type: str
+        ) -> tuple[dict[str, object], object]:
+            return self.logging_hook(kwargs, result, call_type)
+
+    callbacks: Final = [
+        RecordingLogger(),
+        ScreeningGuardrail(default_on=True, event_hook=GuardrailEventHooks.logging_only),
+    ]
+    monkeypatch.setattr(litellm, "success_callback", [])
+    monkeypatch.setattr(litellm, "_async_success_callback", [])
+    monkeypatch.setattr(litellm, "input_callback", [])
+    monkeypatch.setattr(litellm, "turn_off_message_logging", False)
+    now: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
+    logging_obj: Final = LitellmLogging(
+        model="openai/gpt-6.1-sol",
+        messages=[{"role": "user", "content": "screen-this-input"}],
+        stream=stream,
+        call_type="acompletion" if asynchronous else "completion",
+        start_time=now,
+        litellm_call_id="guardrail-redaction",
+        function_id="guardrail-redaction",
+        dynamic_async_success_callbacks=callbacks if asynchronous else [],
+        dynamic_success_callbacks=[] if asynchronous else callbacks,
+    )
+    logging_obj.update_environment_variables(
+        litellm_params={"metadata": {"headers": {"x-litellm-enable-message-redaction": redact}}},
+        optional_params={},
+    )
+    response: Final = ModelResponse(
+        choices=[{"message": {"role": "assistant", "content": "screen-this-output"}}],
+    )
+    logging_obj.post_call(original_response=response)
+    if asynchronous:
+        await logging_obj.async_success_handler(result=response, start_time=now, end_time=now)
+    else:
+        logging_obj.success_handler(result=response, start_time=now, end_time=now)
+
+    assert screened.qsize() == 1
+    guardrail_content: Final = screened.get_nowait()
+    assert "screen-this-input" in guardrail_content
+    assert "screen-this-output" in guardrail_content
+    assert logged.qsize() == 1
+    logger_content: Final = logged.get_nowait()
+    assert ("screen-this-input" in logger_content) is not redact
+    assert ("screen-this-output" in logger_content) is not redact
+    assert response.choices[0].message.content == "screen-this-output"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("call_type", ["acompletion", "aresponses", "anthropic_messages"])
+async def test_logging_guardrail_verdict_survives_message_redaction(
+    monkeypatch: pytest.MonkeyPatch, call_type: str
+) -> None:
+    from queue import SimpleQueue
+
+    from litellm.integrations.custom_guardrail import CustomGuardrail
+    from litellm.types.guardrails import GuardrailEventHooks
+    from litellm.types.utils import GenericGuardrailAPIInputs
+
+    screened: Final[SimpleQueue[tuple[str, tuple[str, ...]]]] = SimpleQueue()
+    payloads: Final[SimpleQueue[str]] = SimpleQueue()
+
+    class ScreeningGuardrail(CustomGuardrail):
+        async def apply_guardrail(
+            self,
+            inputs: GenericGuardrailAPIInputs,
+            request_data: dict[str, object],
+            input_type: Literal["request", "response"],
+            logging_obj: object | None = None,
+        ) -> GenericGuardrailAPIInputs:
+            screened.put((input_type, tuple(inputs.get("texts", []))))
+            self.add_standard_logging_guardrail_information_to_request_data(
+                guardrail_json_response={"detected": True, "texts": inputs.get("texts", [])},
+                request_data=request_data,
+                guardrail_status="guardrail_intervened",
+                event_type=GuardrailEventHooks.logging_only,
+            )
+            return inputs
+
+    class RecordingLogger(CustomLogger):
+        async def async_log_success_event(
+            self,
+            kwargs: dict[str, object],
+            response_obj: object,
+            start_time: datetime.datetime,
+            end_time: datetime.datetime,
+        ) -> None:
+            payloads.put(json.dumps(kwargs["standard_logging_object"]))
+
+    monkeypatch.setattr(litellm, "_async_success_callback", [])
+    monkeypatch.setattr(litellm, "turn_off_message_logging", False)
+    now: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
+    logging_obj: Final = LitellmLogging(
+        model="openai/gpt-6.1-sol",
+        messages=[{"role": "user", "content": "screen-this-input"}],
+        stream=False,
+        call_type=call_type,
+        start_time=now,
+        litellm_call_id="guardrail-verdict",
+        function_id="guardrail-verdict",
+        dynamic_async_success_callbacks=[
+            RecordingLogger(),
+            ScreeningGuardrail(default_on=True, event_hook=GuardrailEventHooks.logging_only),
+        ],
+    )
+    logging_obj.update_environment_variables(
+        litellm_params={"metadata": {"headers": {"x-litellm-enable-message-redaction": True}}},
+        optional_params={},
+    )
+    response: Final = (
+        _responses_api_response_with_text("screen-this-output")
+        if call_type == "aresponses"
+        else ModelResponse(choices=[{"message": {"role": "assistant", "content": "screen-this-output"}}])
+    )
+    await logging_obj.async_success_handler(result=response, start_time=now, end_time=now)
+
+    assert screened.qsize() == 2
+    assert screened.get_nowait() == ("request", ("screen-this-input",))
+    assert screened.get_nowait() == ("response", ("screen-this-output",))
+    assert payloads.qsize() == 1
+    payload: Final = payloads.get_nowait()
+    assert "screen-this-input" not in payload
+    assert "screen-this-output" not in payload
+    assert "guardrail_intervened" in payload
+
+
+def test_sync_logging_guardrail_failure_still_redacts_and_delivers_logs(monkeypatch: pytest.MonkeyPatch) -> None:
+    from queue import SimpleQueue
+
+    from litellm.integrations.custom_guardrail import CustomGuardrail
+    from litellm.types.guardrails import GuardrailEventHooks
+
+    payloads: Final[SimpleQueue[str]] = SimpleQueue()
+
+    class FailingGuardrail(CustomGuardrail):
+        def logging_hook(
+            self, kwargs: dict[str, object], result: object, call_type: str
+        ) -> tuple[dict[str, object], object]:
+            payload: Final = kwargs["standard_logging_object"]
+            assert isinstance(payload, dict)
+            payload["guardrail_information"] = [
+                {"guardrail_status": "guardrail_failed", "guardrail_response": {"text": "private-verdict"}}
+            ]
+            raise RuntimeError("synthetic guardrail failure")
+
+    class RecordingLogger(CustomLogger):
+        def log_success_event(
+            self,
+            kwargs: dict[str, object],
+            response_obj: object,
+            start_time: datetime.datetime,
+            end_time: datetime.datetime,
+        ) -> None:
+            payloads.put(json.dumps([kwargs["messages"], kwargs["standard_logging_object"], str(response_obj)]))
+
+    monkeypatch.setattr(litellm, "success_callback", [])
+    monkeypatch.setattr(litellm, "turn_off_message_logging", False)
+    now: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc)
+    logging_obj: Final = LitellmLogging(
+        model="openai/gpt-6.1-sol",
+        messages=[{"role": "user", "content": "private-input"}],
+        stream=False,
+        call_type="completion",
+        start_time=now,
+        litellm_call_id="guardrail-failure",
+        function_id="guardrail-failure",
+        dynamic_success_callbacks=[
+            FailingGuardrail(default_on=True, event_hook=GuardrailEventHooks.logging_only),
+            RecordingLogger(),
+        ],
+    )
+    logging_obj.update_environment_variables(
+        litellm_params={"metadata": {"headers": {"x-litellm-enable-message-redaction": True}}},
+        optional_params={},
+    )
+    response: Final = ModelResponse(choices=[{"message": {"role": "assistant", "content": "private-output"}}])
+    logging_obj.success_handler(result=response, start_time=now, end_time=now)
+
+    assert payloads.qsize() == 1
+    payload: Final = payloads.get_nowait()
+    assert "private-input" not in payload
+    assert "private-output" not in payload
+    assert "private-verdict" not in payload
+    assert "guardrail_failed" in payload
+    assert "private-input" not in json.dumps(logging_obj.model_call_details["messages"])
+
+
 def test_get_user_agent_tags():
     from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
 

tests/unit/litellm_core_utils/test_redact_messages.py

@@ -51,6 +51,26 @@ SECRET: Final = "private-redaction-canary"
 SIGNATURE: Final = "opaque-thought-signature"
 
 
+@pytest.mark.parametrize("as_list", [False, True])
+def test_redaction_keeps_guardrail_verdict_without_echoing_screened_content(as_list: bool) -> None:
+    entry: Final = {
+        "guardrail_status": "guardrail_intervened",
+        "guardrail_response": {"texts": [SECRET]},
+        "guardrail_request": {"messages": [SECRET]},
+    }
+    entries: Final = [entry] if as_list else entry
+    details: Final = {
+        "litellm_params": {"metadata": {"standard_logging_guardrail_information": entries}},
+        "standard_logging_object": {"guardrail_information": entries},
+    }
+
+    perform_redaction(details, None)
+
+    assert SECRET not in json.dumps(details)
+    assert "guardrail_intervened" in json.dumps(details)
+    assert entry["guardrail_response"] == {"texts": [SECRET]}
+
+
 def _provider_fields() -> dict[str, object]:
     return {
         "reasoning_content": SECRET,