diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index ce57a9bd4a5..8ed83358e6a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1873,8 +1873,9 @@ class Logging(LiteLLMLoggingBaseClass): verbose_logger.exception( "LiteLLM.LoggingError: [Non-Blocking] Exception occurred while logging %s", e ) + post_call_details: Final = self._logging_details_copy(self.model_call_details) redact_message_input_output_from_logging( - model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), + model_call_details=post_call_details, result=original_response, ) # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made @@ -1888,9 +1889,9 @@ class Logging(LiteLLMLoggingBaseClass): if callback == "sentry" and add_breadcrumb: verbose_logger.debug("reaches sentry breadcrumbing") try: - details_to_log = copy.deepcopy(self.model_call_details) + details_to_log = copy.deepcopy(post_call_details) except Exception: - details_to_log = self.model_call_details + details_to_log = post_call_details if litellm.turn_off_message_logging: # make a copy of the _model_Call_details and log it details_to_log.pop("messages", None) @@ -1904,7 +1905,7 @@ class Logging(LiteLLMLoggingBaseClass): ) elif isinstance(callback, CustomLogger): # custom logger class callback.log_post_api_call( - kwargs=self.model_call_details, + kwargs=post_call_details, response_obj=None, start_time=self.start_time, end_time=None, @@ -3111,9 +3112,7 @@ class Logging(LiteLLMLoggingBaseClass): if not build_logging_payload: return None - standard_logging_payload: Final = self._build_standard_logging_payload( - logging_result, start_time, end_time, isolate_call_details=not publish_logging_payload - ) + standard_logging_payload: Final = self._build_standard_logging_payload(logging_result, start_time, end_time) if not publish_logging_payload: return standard_logging_payload self.model_call_details["standard_logging_object"] = standard_logging_payload @@ -3134,17 +3133,26 @@ class Logging(LiteLLMLoggingBaseClass): await invalidate_baseline_cache(self, reason, completed=completed) + def _logging_details_copy( + self, details: Mapping[str, object] + ) -> dict[str, object]: # mutable-ok: legacy redaction mutates its isolated payload + streaming_keys: Final = ("complete_streaming_response", "async_complete_streaming_response") + redact: Final = should_redact_message_logging(self.model_call_details) + return { + **details, + **{key: copy.deepcopy(details[key]) for key in streaming_keys if redact and key in details}, + } + def _build_standard_logging_payload( self, init_response_obj: object, start_time: dt_object, end_time: dt_object, - isolate_call_details: bool = False, ) -> StandardLoggingPayload | None: """Build StandardLoggingPayload and accumulate its construction time.""" _start: Final = time.time() payload: Final = get_standard_logging_object_payload( - kwargs=dict(self.model_call_details) if isolate_call_details else self.model_call_details, + kwargs=self._logging_details_copy(self.model_call_details), init_response_obj=init_response_obj, start_time=start_time, end_time=end_time, @@ -3696,7 +3704,6 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["complete_streaming_response"], start_time, end_time, - isolate_call_details=True, ) if for_async_request and complete_streaming_response is not None else request_logging_payload @@ -3729,35 +3736,47 @@ class Logging(LiteLLMLoggingBaseClass): else callbacks ) - ## REDACT MESSAGES ## - result = _REDACTED_RESULT.validate_python( - redact_message_input_output_from_logging( - model_call_details=call_details, - result=result, - ) - ) ## LOGGING HOOK ## hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks))) + if should_redact_message_logging(call_details) and any( + isinstance(callback, CustomGuardrail) for callback in hook_callbacks + ): + call_details = { # rebind-ok: logging hooks consume and replace this isolated legacy payload + **self._logging_details_copy(call_details), + "messages": copy.deepcopy(call_details.get("messages")), + } + result = copy.deepcopy(result) # rebind-ok: preserve the client response before mutable logging hooks for callback in hook_callbacks: - if isinstance(callback, CustomGuardrail): - from litellm.types.guardrails import GuardrailEventHooks + try: + if isinstance(callback, CustomGuardrail): + from litellm.types.guardrails import GuardrailEventHooks - if ( - callback.should_run_guardrail( - data=call_details, - event_type=GuardrailEventHooks.logging_only, + if ( + callback.should_run_guardrail( + data=call_details, + event_type=GuardrailEventHooks.logging_only, + ) + is not True + ): + continue + + self.shadow_eval_request_snapshot = None + call_details, result = callback.logging_hook( # pyright: ignore[reportUnknownVariableType] # logging_hook is typed with a bare dict + kwargs=call_details, + result=result, + call_type=self.call_type, ) - is not True - ): - continue + except Exception: # noqa: BLE001 # a failed guardrail must not skip redaction or later logging callbacks + verbose_logger.exception("LiteLLM.LoggingError: Exception in guardrail logging_hook") + self._handle_callback_failure(callback=callback) + if not for_async_request: + self.model_call_details = call_details - self.shadow_eval_request_snapshot = None - call_details, result = callback.logging_hook( # pyright: ignore[reportUnknownVariableType] # logging_hook is typed with a bare dict - kwargs=call_details, - result=result, - call_type=self.call_type, - ) - elif isinstance(callback, CustomLogger): + result = _REDACTED_RESULT.validate_python( + redact_message_input_output_from_logging(model_call_details=call_details, result=result) + ) + for callback in hook_callbacks: + if isinstance(callback, CustomLogger) and not isinstance(callback, CustomGuardrail): call_details, result = callback.logging_hook( # pyright: ignore[reportUnknownVariableType] # logging_hook is typed with a bare dict kwargs=call_details, result=result, @@ -4320,16 +4339,17 @@ class Logging(LiteLLMLoggingBaseClass): self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks ) - result = _REDACTED_RESULT.validate_python( - redact_message_input_output_from_logging( - model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}), - result=result, - ) - ) - ## LOGGING HOOK ## hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks))) + if should_redact_message_logging(self.model_call_details) and any( + isinstance(callback, CustomGuardrail) for callback in hook_callbacks + ): + self.model_call_details = { + **self._logging_details_copy(self.model_call_details), + "messages": copy.deepcopy(self.model_call_details.get("messages")), + } + result = copy.deepcopy(result) # rebind-ok: preserve the client response before mutable logging hooks for callback in hook_callbacks: try: if isinstance(callback, CustomGuardrail): @@ -4350,7 +4370,16 @@ class Logging(LiteLLMLoggingBaseClass): result=result, call_type=self.call_type, ) - elif isinstance(callback, CustomLogger): + except Exception: # noqa: BLE001 # a failed guardrail must not skip redaction or later logging callbacks + verbose_logger.exception("LiteLLM.LoggingError: Exception in guardrail async_logging_hook") + self._handle_callback_failure(callback=callback) + + result = _REDACTED_RESULT.validate_python( + redact_message_input_output_from_logging(model_call_details=self.model_call_details, result=result) + ) + for callback in hook_callbacks: + try: + if isinstance(callback, CustomLogger) and not isinstance(callback, CustomGuardrail): result = redact_message_input_output_from_custom_logger( result=result, litellm_logging_obj=self, custom_logger=callback ) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index ac5c0720eff..bd0e063b7a6 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -22,6 +22,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.classifier_logging import without_classifier_audit from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, + redact_nested_match_and_regex_keys, # pyright: ignore[reportUnknownVariableType] # shared legacy helper uses bare dicts ) from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items @@ -31,7 +32,7 @@ from litellm.llms.vertex_ai.common_utils import ( ) from litellm.secret_managers.main import str_to_bool from litellm.types.router import BaselineRouteStamp -from litellm.types.utils import StandardCallbackDynamicParams +from litellm.types.utils import PROMPT_CARRYING_GUARDRAIL_FIELDS, StandardCallbackDynamicParams if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import ( @@ -432,6 +433,10 @@ def _redact_standard_logging_object( if standard_logging_object.get("messages") is not None: standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}] + if "guardrail_information" in standard_logging_object: + standard_logging_object["guardrail_information"] = _redacted_guardrail_information( + standard_logging_object["guardrail_information"] + ) model_parameters: Final = _as_mapping(standard_logging_object.get("model_parameters")) if model_parameters is not None and redact_instructions: standard_logging_object["model_parameters"] = _with_redacted_instructions(model_parameters) @@ -492,11 +497,33 @@ def _redact_model_response_dict_choices(choices: Iterable[object], redacted_str: _redact_choice_content(choice) -def _redacted_baseline_metadata(metadata: Mapping[str, object]) -> Mapping[str, object]: +def _redacted_guardrail_information(value: object) -> object: + if not isinstance(value, (dict, list)): + return value + return redact_nested_match_and_regex_keys(value, keys=PROMPT_CARRYING_GUARDRAIL_FIELDS) # pyright: ignore[reportUnknownVariableType] # opaque result of the legacy redactor + + +def _redacted_logging_metadata(metadata: Mapping[str, object]) -> Mapping[str, object]: route: Final = metadata.get("_autorouter_baseline_route") - if not isinstance(route, BaselineRouteStamp): + if not isinstance(route, BaselineRouteStamp) and "standard_logging_guardrail_information" not in metadata: return metadata - return {**metadata, "_autorouter_baseline_route": replace(route, request_parameters=None)} + return { + **metadata, + **( + {"_autorouter_baseline_route": replace(route, request_parameters=None)} + if isinstance(route, BaselineRouteStamp) + else {} + ), + **( + { + "standard_logging_guardrail_information": _redacted_guardrail_information( + metadata["standard_logging_guardrail_information"] + ) + } + if "standard_logging_guardrail_information" in metadata + else {} + ), + } def _redacted_proxy_server_request(request: Mapping[str, object], *, redact_body: bool) -> Mapping[str, object]: @@ -533,7 +560,7 @@ def redacted_litellm_params(params: Mapping[str, object], *, redact_request_body return { **(_with_redacted_instructions(params) if redact_request_body else params), **{ - key: _redacted_baseline_metadata(value) + key: _redacted_logging_metadata(value) for key, value in params.items() if key in ("metadata", "litellm_metadata") and isinstance(value, Mapping) }, diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index e8ea0c04f6f..97a00417a54 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -3632,6 +3632,230 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g assert "snapshot-only" not in json.dumps(logging_obj.model_call_details, default=str) +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("redact", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +async def test_logging_guardrail_screens_original_content_while_loggers_honor_redaction( + monkeypatch: pytest.MonkeyPatch, asynchronous: bool, redact: bool, stream: bool +) -> None: + from queue import SimpleQueue + + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + screened: Final[SimpleQueue[str]] = SimpleQueue() + logged: Final[SimpleQueue[str]] = SimpleQueue() + + class ScreeningGuardrail(CustomGuardrail): + def logging_hook( + self, kwargs: dict[str, object], result: object, call_type: str + ) -> tuple[dict[str, object], object]: + screened.put(json.dumps([kwargs["messages"], str(result)])) + if redact and isinstance(result, ModelResponse): + result.choices[0].message.content = "masked-by-guardrail" + return kwargs, result + + async def async_logging_hook( + self, kwargs: dict[str, object], result: object, call_type: str + ) -> tuple[dict[str, object], object]: + return self.logging_hook(kwargs, result, call_type) + + class RecordingLogger(CustomLogger): + def logging_hook( + self, kwargs: dict[str, object], result: object, call_type: str + ) -> tuple[dict[str, object], object]: + logged.put(json.dumps([kwargs["messages"], str(result)])) + return kwargs, result + + async def async_logging_hook( + self, kwargs: dict[str, object], result: object, call_type: str + ) -> tuple[dict[str, object], object]: + return self.logging_hook(kwargs, result, call_type) + + callbacks: Final = [ + RecordingLogger(), + ScreeningGuardrail(default_on=True, event_hook=GuardrailEventHooks.logging_only), + ] + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "input_callback", []) + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + now: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + logging_obj: Final = LitellmLogging( + model="openai/gpt-6.1-sol", + messages=[{"role": "user", "content": "screen-this-input"}], + stream=stream, + call_type="acompletion" if asynchronous else "completion", + start_time=now, + litellm_call_id="guardrail-redaction", + function_id="guardrail-redaction", + dynamic_async_success_callbacks=callbacks if asynchronous else [], + dynamic_success_callbacks=[] if asynchronous else callbacks, + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {"headers": {"x-litellm-enable-message-redaction": redact}}}, + optional_params={}, + ) + response: Final = ModelResponse( + choices=[{"message": {"role": "assistant", "content": "screen-this-output"}}], + ) + logging_obj.post_call(original_response=response) + if asynchronous: + await logging_obj.async_success_handler(result=response, start_time=now, end_time=now) + else: + logging_obj.success_handler(result=response, start_time=now, end_time=now) + + assert screened.qsize() == 1 + guardrail_content: Final = screened.get_nowait() + assert "screen-this-input" in guardrail_content + assert "screen-this-output" in guardrail_content + assert logged.qsize() == 1 + logger_content: Final = logged.get_nowait() + assert ("screen-this-input" in logger_content) is not redact + assert ("screen-this-output" in logger_content) is not redact + assert response.choices[0].message.content == "screen-this-output" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["acompletion", "aresponses", "anthropic_messages"]) +async def test_logging_guardrail_verdict_survives_message_redaction( + monkeypatch: pytest.MonkeyPatch, call_type: str +) -> None: + from queue import SimpleQueue + + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + from litellm.types.utils import GenericGuardrailAPIInputs + + screened: Final[SimpleQueue[tuple[str, tuple[str, ...]]]] = SimpleQueue() + payloads: Final[SimpleQueue[str]] = SimpleQueue() + + class ScreeningGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: object | None = None, + ) -> GenericGuardrailAPIInputs: + screened.put((input_type, tuple(inputs.get("texts", [])))) + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response={"detected": True, "texts": inputs.get("texts", [])}, + request_data=request_data, + guardrail_status="guardrail_intervened", + event_type=GuardrailEventHooks.logging_only, + ) + return inputs + + class RecordingLogger(CustomLogger): + async def async_log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + payloads.put(json.dumps(kwargs["standard_logging_object"])) + + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + now: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + logging_obj: Final = LitellmLogging( + model="openai/gpt-6.1-sol", + messages=[{"role": "user", "content": "screen-this-input"}], + stream=False, + call_type=call_type, + start_time=now, + litellm_call_id="guardrail-verdict", + function_id="guardrail-verdict", + dynamic_async_success_callbacks=[ + RecordingLogger(), + ScreeningGuardrail(default_on=True, event_hook=GuardrailEventHooks.logging_only), + ], + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {"headers": {"x-litellm-enable-message-redaction": True}}}, + optional_params={}, + ) + response: Final = ( + _responses_api_response_with_text("screen-this-output") + if call_type == "aresponses" + else ModelResponse(choices=[{"message": {"role": "assistant", "content": "screen-this-output"}}]) + ) + await logging_obj.async_success_handler(result=response, start_time=now, end_time=now) + + assert screened.qsize() == 2 + assert screened.get_nowait() == ("request", ("screen-this-input",)) + assert screened.get_nowait() == ("response", ("screen-this-output",)) + assert payloads.qsize() == 1 + payload: Final = payloads.get_nowait() + assert "screen-this-input" not in payload + assert "screen-this-output" not in payload + assert "guardrail_intervened" in payload + + +def test_sync_logging_guardrail_failure_still_redacts_and_delivers_logs(monkeypatch: pytest.MonkeyPatch) -> None: + from queue import SimpleQueue + + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + payloads: Final[SimpleQueue[str]] = SimpleQueue() + + class FailingGuardrail(CustomGuardrail): + def logging_hook( + self, kwargs: dict[str, object], result: object, call_type: str + ) -> tuple[dict[str, object], object]: + payload: Final = kwargs["standard_logging_object"] + assert isinstance(payload, dict) + payload["guardrail_information"] = [ + {"guardrail_status": "guardrail_failed", "guardrail_response": {"text": "private-verdict"}} + ] + raise RuntimeError("synthetic guardrail failure") + + class RecordingLogger(CustomLogger): + def log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + payloads.put(json.dumps([kwargs["messages"], kwargs["standard_logging_object"], str(response_obj)])) + + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + now: Final = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + logging_obj: Final = LitellmLogging( + model="openai/gpt-6.1-sol", + messages=[{"role": "user", "content": "private-input"}], + stream=False, + call_type="completion", + start_time=now, + litellm_call_id="guardrail-failure", + function_id="guardrail-failure", + dynamic_success_callbacks=[ + FailingGuardrail(default_on=True, event_hook=GuardrailEventHooks.logging_only), + RecordingLogger(), + ], + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {"headers": {"x-litellm-enable-message-redaction": True}}}, + optional_params={}, + ) + response: Final = ModelResponse(choices=[{"message": {"role": "assistant", "content": "private-output"}}]) + logging_obj.success_handler(result=response, start_time=now, end_time=now) + + assert payloads.qsize() == 1 + payload: Final = payloads.get_nowait() + assert "private-input" not in payload + assert "private-output" not in payload + assert "private-verdict" not in payload + assert "guardrail_failed" in payload + assert "private-input" not in json.dumps(logging_obj.model_call_details["messages"]) + + def test_get_user_agent_tags(): from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 491ea5315ff..a476244e2f3 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -51,6 +51,26 @@ SECRET: Final = "private-redaction-canary" SIGNATURE: Final = "opaque-thought-signature" +@pytest.mark.parametrize("as_list", [False, True]) +def test_redaction_keeps_guardrail_verdict_without_echoing_screened_content(as_list: bool) -> None: + entry: Final = { + "guardrail_status": "guardrail_intervened", + "guardrail_response": {"texts": [SECRET]}, + "guardrail_request": {"messages": [SECRET]}, + } + entries: Final = [entry] if as_list else entry + details: Final = { + "litellm_params": {"metadata": {"standard_logging_guardrail_information": entries}}, + "standard_logging_object": {"guardrail_information": entries}, + } + + perform_redaction(details, None) + + assert SECRET not in json.dumps(details) + assert "guardrail_intervened" in json.dumps(details) + assert entry["guardrail_response"] == {"texts": [SECRET]} + + def _provider_fields() -> dict[str, object]: return { "reasoning_content": SECRET,