From 954cf204b1d0975b0535ac58e72aec10c0b90fbd Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:44:51 -0800 Subject: [PATCH] fix(logging): drop chat completion logprobs when message logging is off Bridged /v1/responses calls log the chat completion, whose choices[].logprobs spelled out the redacted answer --- litellm/litellm_core_utils/redact_messages.py | 7 ++-- .../test_redact_messages.py | 34 +++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 24f2476391b..5b5a5cc15df 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -206,6 +206,7 @@ def _redact_choice_content(choice): """Helper to redact content in a choice (message or delta).""" if isinstance(choice, (litellm.Choices, litellm.utils.StreamingChoices, litellm.utils.TextChoices)): drop_token_ids(choice) + _drop_field_when_present(choice, "logprobs") _redact_provider_specific_fields(choice) if isinstance(choice, litellm.Choices): if choice.message.content is not None: @@ -232,7 +233,6 @@ def _redact_choice_content(choice): elif isinstance(choice, litellm.utils.TextChoices): _redact_field_when_present(choice, "text") _redact_field_when_present(choice, "reasoning_content") - _drop_field_when_present(choice, "logprobs") def _as_mapping(value: object) -> Mapping[str, object] | None: @@ -465,8 +465,9 @@ def _redact_model_response_dict_choices(choices: Iterable[object], redacted_str: for choice in choices: if isinstance(choice, dict): _redact_provider_specific_fields_dict(choice) - if choice.get("token_ids") is not None: - choice["token_ids"] = None + for field in ("token_ids", "logprobs"): + if choice.get(field) is not None: + choice[field] = None if ( body := next((choice[key] for key in ("message", "delta") if isinstance(choice.get(key), dict)), None) ) is not None: diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index dc3fdc26612..58fd546c6dc 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -163,6 +163,40 @@ def test_an_opted_out_callback_gets_a_text_completion_with_its_answer_redacted() assert response.choices[0].text == SECRET, "the caller's own response is not redacted" +def _chat_logprobs() -> dict[str, object]: + top: Final = {"token": SECRET, "logprob": -0.2, "bytes": list(SECRET.encode())} + return {"content": [{"token": SECRET, "logprob": -0.1, "bytes": list(SECRET.encode()), "top_logprobs": [top]}]} + + +def _chat_choice_with_logprobs() -> dict[str, object]: + return {"index": 0, "message": {"role": "assistant", "content": "answer"}, "logprobs": _chat_logprobs()} + + +def _delta_choice_with_logprobs() -> dict[str, object]: + return {"index": 0, "delta": {"content": "answer"}, "logprobs": _chat_logprobs()} + + +@pytest.mark.parametrize( + ("details", "result"), + [ + pytest.param({}, ModelResponse(choices=[_chat_choice_with_logprobs()]), id="message_object"), + pytest.param({}, ModelResponse(choices=[_chat_choice_with_logprobs()]).model_dump(), id="message_dict"), + pytest.param({}, {"choices": [_delta_choice_with_logprobs()]}, id="delta_dict"), + pytest.param( + {"stream": True, "complete_streaming_response": ModelResponseStream(choices=[_delta_choice_with_logprobs()])}, + None, + id="streamed", + ), + ], +) +def test_redaction_hides_the_answer_tokens_in_chat_logprobs(details: dict[str, object], result: object) -> None: + assert SECRET in str(details) + str(result) + + redacted: Final = perform_redaction(details, result) + + assert SECRET not in str(redacted) and SECRET not in str(details), (redacted, details) + + def test_redaction_redacts_reasoning_copied_into_hidden_params_in_place() -> None: hidden_params: Final = {"model_id": "deployment", "provider_specific_fields": _provider_fields()} original_hidden_params: Final = copy.deepcopy(hidden_params)