scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-23 05:34:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 24

fix(deepeval): redact response text when turn_off_message_logging is set

Browse source 
litellm only applies per-callback redaction on the async success path, so a
sync completion() or responses() call with DeepEvalLogger(turn_off_message_logging=True)
sent the assistant text to Confident AI. Redact the text in the logger, the
same way tool arguments already are. Also drop two comments that restated the code.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
KardeniaPoyu 2026-10-07 20:54:39 +08:00
parent f522468e0e
commit d29adb699a
2 changed files with 37 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
litellm/integrations/deepeval/deepeval.py
View file

@ -68,64 +68,61 @@ def _content_text(content: object) -> str:
return "".join(text for part in _dict_items(content) if (text := _part_text(part)) is not None)
def _tool_call(call_id: object, name: object, arguments: object, redact_arguments: bool) -> _ToolCall:
# Per-callback redaction (turn_off_message_logging) blanks response text but not tool arguments.
def _tool_call(call_id: object, name: object, arguments: object, redact: bool) -> _ToolCall:
function: Final[_ToolFunction] = {
"name": _as_str(name) or "",
"arguments": _REDACTED if redact_arguments else (_as_str(arguments) or ""),
"arguments": _REDACTED if redact else (_as_str(arguments) or ""),
}
tool_call: Final[_ToolCall] = {"id": _as_str(call_id) or "", "type": "function", "function": function}
return tool_call
def _chat_tool_call(tool_call: Mapping[str, object], redact_arguments: bool) -> _ToolCall:
def _chat_tool_call(tool_call: Mapping[str, object], redact: bool) -> _ToolCall:
function: Final = _as_mapping(tool_call.get("function")) or {}
return _tool_call(tool_call.get("id"), function.get("name"), function.get("arguments"), redact_arguments)
return _tool_call(tool_call.get("id"), function.get("name"), function.get("arguments"), redact)
def _responses_tool_call(item: Mapping[str, object], redact_arguments: bool) -> _ToolCall:
def _responses_tool_call(item: Mapping[str, object], redact: bool) -> _ToolCall:
arguments_key: Final = "input" if item.get("type") == "custom_tool_call" else "arguments"
call_id: Final = item.get("call_id") or item.get("id")
return _tool_call(call_id, item.get("name"), item.get(arguments_key), redact_arguments)
return _tool_call(call_id, item.get("name"), item.get(arguments_key), redact)
def _span_output(text: str, tool_calls: tuple[_ToolCall, ...]) -> _AssistantMessage | str:
def _span_output(text: str, tool_calls: tuple[_ToolCall, ...], redact: bool) -> _AssistantMessage | str:
span_text: Final = _REDACTED if redact and text else text
if not tool_calls:
return text
message: Final[_AssistantMessage] = {"role": "assistant", "content": text or None, "tool_calls": tool_calls}
return span_text
message: Final[_AssistantMessage] = {"role": "assistant", "content": span_text or None, "tool_calls": tool_calls}
return message
def _chat_output(choice: Mapping[str, object], redact_arguments: bool) -> _AssistantMessage | str | None:
def _chat_output(choice: Mapping[str, object], redact: bool) -> _AssistantMessage | str | None:
message: Final = _as_mapping(choice.get("message")) or {}
content: Final = message.get("content")
raw_tool_calls: Final = _dict_items(message.get("tool_calls"))
tool_calls: Final = tuple(_chat_tool_call(tool_call, redact_arguments) for tool_call in raw_tool_calls)
tool_calls: Final = tuple(_chat_tool_call(tool_call, redact) for tool_call in raw_tool_calls)
if content is None and not tool_calls:
return None
return _span_output(_content_text(content), tool_calls)
return _span_output(_content_text(content), tool_calls, redact)
def _responses_output(
output_items: tuple[Mapping[str, object], ...], redact_arguments: bool
) -> _AssistantMessage | str:
def _responses_output(output_items: tuple[Mapping[str, object], ...], redact: bool) -> _AssistantMessage | str:
messages: Final = tuple(item for item in output_items if item.get("type") == "message")
message_texts: Final = tuple(_content_text(item.get("content")) for item in messages)
tool_items: Final = tuple(item for item in output_items if item.get("type") in _RESPONSES_TOOL_CALL_TYPES)
tool_calls: Final = tuple(_responses_tool_call(item, redact_arguments) for item in tool_items)
# Blank line between messages so e.g. commentary and final answer don't run together.
tool_calls: Final = tuple(_responses_tool_call(item, redact) for item in tool_items)
text: Final = "\n\n".join(message_text for message_text in message_texts if message_text)
return _span_output(text, tool_calls) if text or tool_calls else "NO_OUTPUT"
return _span_output(text, tool_calls, redact) if text or tool_calls else "NO_OUTPUT"
def _success_output(response: object, redact_arguments: bool) -> _AssistantMessage | str | None:
def _success_output(response: object, redact: bool) -> _AssistantMessage | str | None:
"""Chat completions read ``choices``; Responses API payloads read ``output``."""
response_dict: Final = _as_mapping(response) or {}
choices: Final = _dict_items(response_dict.get("choices"))
if choices:
return _chat_output(choices[0], redact_arguments)
return _chat_output(choices[0], redact)
output_items: Final = _dict_items(response_dict.get("output"))
return _responses_output(output_items, redact_arguments) if output_items else "NO_OUTPUT"
return _responses_output(output_items, redact) if output_items else "NO_OUTPUT"
# This file includes the custom callbacks for LiteLLM Proxy
@ -212,9 +209,7 @@ class DeepEvalLogger(CustomLogger):
# extract usage
usage: Final = standard_logging_object.get("response", {}).get("usage", {})
if is_success:
output = _success_output(
standard_logging_object.get("response", {}), redact_arguments=self.turn_off_message_logging
)
output = _success_output(standard_logging_object.get("response", {}), redact=self.turn_off_message_logging)
else:
output = str(standard_logging_object.get("error_string", ""))
return BaseApiSpan(

17
tests/unit/integrations/test_deepeval.py
View file

@ -215,6 +215,23 @@ class TestDeepEvalLogger(unittest.TestCase):
self.assertEqual(chat_output, {"role": "assistant", "content": None, "tool_calls": [redacted_tool_call]})
self.assertEqual(responses_output, {"role": "assistant", "content": None, "tool_calls": [redacted_tool_call]})
def test_turn_off_message_logging_redacts_text(self):
self.logger.turn_off_message_logging = True
chat_text: Final = self._sent_span_output(_chat_response(content="secret answer", tool_calls=()))
responses_text: Final = self._sent_span_output({"usage": {}, "output": [_responses_message("secret answer")]})
chat_mixed: Final = self._sent_span_output(
_chat_response(content="secret answer", tool_calls=(_CHAT_TOOL_CALL,))
)
responses_mixed: Final = self._sent_span_output(
{"usage": {}, "output": [_responses_message("secret answer"), _RESPONSES_FUNCTION_CALL]}
)
self.assertEqual(chat_text, "redacted-by-litellm")
self.assertEqual(responses_text, "redacted-by-litellm")
for mixed in (chat_mixed, responses_mixed):
self.assertEqual(mixed["content"], "redacted-by-litellm")
self.assertEqual(mixed["tool_calls"][0]["function"]["arguments"], "redacted-by-litellm")
def test_empty_choices_records_no_output(self):
output: Final = self._sent_span_output({"usage": {}, "choices": []})