From d29adb699a603540293b7d1dee38868010f9e1fb Mon Sep 17 00:00:00 2001 From: KardeniaPoyu Date: Wed, 7 Oct 2026 20:54:39 +0800 Subject: [PATCH] fix(deepeval): redact response text when turn_off_message_logging is set litellm only applies per-callback redaction on the async success path, so a sync completion() or responses() call with DeepEvalLogger(turn_off_message_logging=True) sent the assistant text to Confident AI. Redact the text in the logger, the same way tool arguments already are. Also drop two comments that restated the code. Co-Authored-By: Claude Opus 5.5 --- litellm/integrations/deepeval/deepeval.py | 45 ++++++++++------------- tests/unit/integrations/test_deepeval.py | 17 +++++++++ 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/litellm/integrations/deepeval/deepeval.py b/litellm/integrations/deepeval/deepeval.py index 7a5708326c2..eafd07ab241 100644 --- a/litellm/integrations/deepeval/deepeval.py +++ b/litellm/integrations/deepeval/deepeval.py @@ -68,64 +68,61 @@ def _content_text(content: object) -> str: return "".join(text for part in _dict_items(content) if (text := _part_text(part)) is not None) -def _tool_call(call_id: object, name: object, arguments: object, redact_arguments: bool) -> _ToolCall: - # Per-callback redaction (turn_off_message_logging) blanks response text but not tool arguments. +def _tool_call(call_id: object, name: object, arguments: object, redact: bool) -> _ToolCall: function: Final[_ToolFunction] = { "name": _as_str(name) or "", - "arguments": _REDACTED if redact_arguments else (_as_str(arguments) or ""), + "arguments": _REDACTED if redact else (_as_str(arguments) or ""), } tool_call: Final[_ToolCall] = {"id": _as_str(call_id) or "", "type": "function", "function": function} return tool_call -def _chat_tool_call(tool_call: Mapping[str, object], redact_arguments: bool) -> _ToolCall: +def _chat_tool_call(tool_call: Mapping[str, object], redact: bool) -> _ToolCall: function: Final = _as_mapping(tool_call.get("function")) or {} - return _tool_call(tool_call.get("id"), function.get("name"), function.get("arguments"), redact_arguments) + return _tool_call(tool_call.get("id"), function.get("name"), function.get("arguments"), redact) -def _responses_tool_call(item: Mapping[str, object], redact_arguments: bool) -> _ToolCall: +def _responses_tool_call(item: Mapping[str, object], redact: bool) -> _ToolCall: arguments_key: Final = "input" if item.get("type") == "custom_tool_call" else "arguments" call_id: Final = item.get("call_id") or item.get("id") - return _tool_call(call_id, item.get("name"), item.get(arguments_key), redact_arguments) + return _tool_call(call_id, item.get("name"), item.get(arguments_key), redact) -def _span_output(text: str, tool_calls: tuple[_ToolCall, ...]) -> _AssistantMessage | str: +def _span_output(text: str, tool_calls: tuple[_ToolCall, ...], redact: bool) -> _AssistantMessage | str: + span_text: Final = _REDACTED if redact and text else text if not tool_calls: - return text - message: Final[_AssistantMessage] = {"role": "assistant", "content": text or None, "tool_calls": tool_calls} + return span_text + message: Final[_AssistantMessage] = {"role": "assistant", "content": span_text or None, "tool_calls": tool_calls} return message -def _chat_output(choice: Mapping[str, object], redact_arguments: bool) -> _AssistantMessage | str | None: +def _chat_output(choice: Mapping[str, object], redact: bool) -> _AssistantMessage | str | None: message: Final = _as_mapping(choice.get("message")) or {} content: Final = message.get("content") raw_tool_calls: Final = _dict_items(message.get("tool_calls")) - tool_calls: Final = tuple(_chat_tool_call(tool_call, redact_arguments) for tool_call in raw_tool_calls) + tool_calls: Final = tuple(_chat_tool_call(tool_call, redact) for tool_call in raw_tool_calls) if content is None and not tool_calls: return None - return _span_output(_content_text(content), tool_calls) + return _span_output(_content_text(content), tool_calls, redact) -def _responses_output( - output_items: tuple[Mapping[str, object], ...], redact_arguments: bool -) -> _AssistantMessage | str: +def _responses_output(output_items: tuple[Mapping[str, object], ...], redact: bool) -> _AssistantMessage | str: messages: Final = tuple(item for item in output_items if item.get("type") == "message") message_texts: Final = tuple(_content_text(item.get("content")) for item in messages) tool_items: Final = tuple(item for item in output_items if item.get("type") in _RESPONSES_TOOL_CALL_TYPES) - tool_calls: Final = tuple(_responses_tool_call(item, redact_arguments) for item in tool_items) - # Blank line between messages so e.g. commentary and final answer don't run together. + tool_calls: Final = tuple(_responses_tool_call(item, redact) for item in tool_items) text: Final = "\n\n".join(message_text for message_text in message_texts if message_text) - return _span_output(text, tool_calls) if text or tool_calls else "NO_OUTPUT" + return _span_output(text, tool_calls, redact) if text or tool_calls else "NO_OUTPUT" -def _success_output(response: object, redact_arguments: bool) -> _AssistantMessage | str | None: +def _success_output(response: object, redact: bool) -> _AssistantMessage | str | None: """Chat completions read ``choices``; Responses API payloads read ``output``.""" response_dict: Final = _as_mapping(response) or {} choices: Final = _dict_items(response_dict.get("choices")) if choices: - return _chat_output(choices[0], redact_arguments) + return _chat_output(choices[0], redact) output_items: Final = _dict_items(response_dict.get("output")) - return _responses_output(output_items, redact_arguments) if output_items else "NO_OUTPUT" + return _responses_output(output_items, redact) if output_items else "NO_OUTPUT" # This file includes the custom callbacks for LiteLLM Proxy @@ -212,9 +209,7 @@ class DeepEvalLogger(CustomLogger): # extract usage usage: Final = standard_logging_object.get("response", {}).get("usage", {}) if is_success: - output = _success_output( - standard_logging_object.get("response", {}), redact_arguments=self.turn_off_message_logging - ) + output = _success_output(standard_logging_object.get("response", {}), redact=self.turn_off_message_logging) else: output = str(standard_logging_object.get("error_string", "")) return BaseApiSpan( diff --git a/tests/unit/integrations/test_deepeval.py b/tests/unit/integrations/test_deepeval.py index 3222590e623..02517b8c870 100644 --- a/tests/unit/integrations/test_deepeval.py +++ b/tests/unit/integrations/test_deepeval.py @@ -215,6 +215,23 @@ class TestDeepEvalLogger(unittest.TestCase): self.assertEqual(chat_output, {"role": "assistant", "content": None, "tool_calls": [redacted_tool_call]}) self.assertEqual(responses_output, {"role": "assistant", "content": None, "tool_calls": [redacted_tool_call]}) + def test_turn_off_message_logging_redacts_text(self): + self.logger.turn_off_message_logging = True + chat_text: Final = self._sent_span_output(_chat_response(content="secret answer", tool_calls=())) + responses_text: Final = self._sent_span_output({"usage": {}, "output": [_responses_message("secret answer")]}) + chat_mixed: Final = self._sent_span_output( + _chat_response(content="secret answer", tool_calls=(_CHAT_TOOL_CALL,)) + ) + responses_mixed: Final = self._sent_span_output( + {"usage": {}, "output": [_responses_message("secret answer"), _RESPONSES_FUNCTION_CALL]} + ) + + self.assertEqual(chat_text, "redacted-by-litellm") + self.assertEqual(responses_text, "redacted-by-litellm") + for mixed in (chat_mixed, responses_mixed): + self.assertEqual(mixed["content"], "redacted-by-litellm") + self.assertEqual(mixed["tool_calls"][0]["function"]["arguments"], "redacted-by-litellm") + def test_empty_choices_records_no_output(self): output: Final = self._sent_span_output({"usage": {}, "choices": []})