From 50547e12dc048500538136e5400f3e781198deca Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:58:46 -0800 Subject: [PATCH] fix(logging): redact chat refusals when message logging is off Message logging redaction replaced content and reasoning but left message.refusal and delta.refusal in logged chat responses, so a refusal reached loggers in full. Both the object and dict choice paths now redact refusal, and the dict path uses one field list for text and cleared fields. --- litellm/litellm_core_utils/redact_messages.py | 49 +++++++------- .../test_redact_messages.py | 67 +++++++++++++++++++ 2 files changed, 91 insertions(+), 25 deletions(-) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 981859c620b..a77fd582dbc 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -201,6 +201,7 @@ def _redact_choice_content(choice): choice.message.content = REDACTED_BY_LITELLM if getattr(choice.message, "reasoning_content", None) is not None: choice.message.reasoning_content = REDACTED_BY_LITELLM + _redact_field_when_present(choice.message, "refusal") if hasattr(choice.message, "thinking_blocks"): choice.message.thinking_blocks = None _redact_provider_specific_fields(choice.message) @@ -211,6 +212,7 @@ def _redact_choice_content(choice): choice.delta.content = REDACTED_BY_LITELLM if getattr(choice.delta, "reasoning_content", None) is not None: choice.delta.reasoning_content = REDACTED_BY_LITELLM + _redact_field_when_present(choice.delta, "refusal") if hasattr(choice.delta, "thinking_blocks"): choice.delta.thinking_blocks = None _redact_provider_specific_fields(choice.delta) @@ -418,34 +420,31 @@ def _redact_tool_calls_dict(message: Mapping[str, object]) -> None: function_call["arguments"] = REDACTED_BY_LITELLM +_REDACTED_TEXT_FIELDS: Final = ("content", "reasoning_content", "refusal") +_CLEARED_FIELDS: Final = ("thinking_blocks", "audio") + + +def _choice_payload(choice: Mapping[str, object]) -> object: + return choice["message"] if isinstance(choice.get("message"), dict) else choice.get("delta") + + def _redact_model_response_dict_choices(choices, redacted_str: str): for choice in choices: - if isinstance(choice, dict): - _redact_provider_specific_fields_dict(choice) - if "message" in choice and isinstance(choice["message"], dict): - if choice["message"].get("content") is not None: - choice["message"]["content"] = redacted_str - if choice["message"].get("reasoning_content") is not None: - choice["message"]["reasoning_content"] = redacted_str - if "thinking_blocks" in choice["message"]: - choice["message"]["thinking_blocks"] = None - if "audio" in choice["message"]: - choice["message"]["audio"] = None - _redact_tool_calls_dict(choice["message"]) - elif "delta" in choice and isinstance(choice["delta"], dict): - if choice["delta"].get("content") is not None: - choice["delta"]["content"] = redacted_str - if choice["delta"].get("reasoning_content") is not None: - choice["delta"]["reasoning_content"] = redacted_str - if "thinking_blocks" in choice["delta"]: - choice["delta"]["thinking_blocks"] = None - if "audio" in choice["delta"]: - choice["delta"]["audio"] = None - _redact_tool_calls_dict(choice["delta"]) - elif choice.get("text") is not None: - choice["text"] = redacted_str - else: + if not isinstance(choice, dict): _redact_choice_content(choice) + continue + _redact_provider_specific_fields_dict(choice) + if not isinstance(payload := _choice_payload(choice), dict): + if choice.get("text") is not None: + choice["text"] = redacted_str + continue + for text_field in _REDACTED_TEXT_FIELDS: + if payload.get(text_field) is not None: + payload[text_field] = redacted_str + for cleared_field in _CLEARED_FIELDS: + if cleared_field in payload: + payload[cleared_field] = None + _redact_tool_calls_dict(payload) def _redacted_baseline_metadata(metadata: Mapping[str, object]) -> Mapping[str, object]: diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 9e385c588ae..6c8f0545760 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -20,6 +20,7 @@ from unittest.mock import patch import httpx import pytest import pytest_asyncio +from openai import AsyncOpenAI from pydantic import JsonValue import litellm @@ -1654,6 +1655,72 @@ async def test_responses_redaction_removes_request_instructions_from_logged_kwar assert SECRET not in json.dumps({key: kwargs.get(key) for key in logged_keys}, default=str) +_REFUSAL_PIECES: Final = ("I can't share ", SECRET) + + +def _chat_chunk(delta: Mapping[str, object], finish_reason: str | None) -> str: + chunk: Final = { + "id": "chatcmpl-refusal", + "object": "chat.completion.chunk", + "created": 0, + "model": "gpt-4.1", + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + return f"data: {json.dumps(chunk)}\n\n" + + +def _openai_refusal_reply(request: httpx.Request) -> httpx.Response: + if json.loads(request.content).get("stream"): + deltas: Final = ({"role": "assistant", "refusal": ""}, *({"refusal": piece} for piece in _REFUSAL_PIECES)) + sse: Final = "".join(_chat_chunk(delta, None) for delta in deltas) + _chat_chunk({}, "stop") + "data: [DONE]\n\n" + return httpx.Response(200, request=request, headers={"content-type": "text/event-stream"}, content=sse) + message: Final = {"role": "assistant", "content": None, "refusal": "".join(_REFUSAL_PIECES)} + return httpx.Response( + 200, + request=request, + json={ + "id": "chatcmpl-refusal", + "object": "chat.completion", + "created": 0, + "model": "gpt-4.1", + "choices": [{"index": 0, "message": message, "finish_reason": "stop"}], + }, + ) + + +async def _caller_refusal(response: object) -> str | None: + if isinstance(response, litellm.CustomStreamWrapper): + return "".join([getattr(chunk.choices[0].delta, "refusal", None) or "" async for chunk in response]) + assert isinstance(response, ModelResponse) + return getattr(response.choices[0].message, "refusal", None) # pyright: ignore[reportAttributeAccessIssue] # choices is a Choices | StreamingChoices union + + +@pytest.mark.usefixtures("drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [True, False]) +async def test_redaction_hides_a_refusal_from_loggers_but_not_from_the_caller( + stream: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + test_custom_logger: Final = TestCustomLogger() + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + monkeypatch.setattr(litellm, "callbacks", [test_custom_logger]) + client: Final = AsyncOpenAI( + api_key="fake-key", http_client=httpx.AsyncClient(transport=httpx.MockTransport(_openai_refusal_reply)) + ) + + response: Final = await litellm.acompletion( + model="openai/gpt-4.1", messages=[{"role": "user", "content": "hi"}], stream=stream, client=client + ) + caller_refusal: Final = await _caller_refusal(response) + + await asyncio.wait_for(test_custom_logger.logged.wait(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS) + logged: Final = test_custom_logger.logged_standard_logging_payload + assert caller_refusal == "".join(_REFUSAL_PIECES) + assert logged is not None + assert logged["response"]["choices"][0]["message"]["refusal"] == "redacted-by-litellm" + assert SECRET not in json.dumps(logged["response"], default=str) + + @pytest.mark.usefixtures("drain_logging_worker", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_one_callback_opting_out_redacts_instructions_only_for_itself(