Merge 73003559e6 into f285229b51

litellm/integrations/custom_logger.py

@@ -940,20 +940,11 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
                 response: Final = standard_logging_object_copy["response"]
                 # Check if this is a ResponsesAPIResponse (has "output" field)
                 if isinstance(response, dict) and "output" in response:
-                    # Make a copy to avoid modifying the original
-                    from copy import deepcopy
+                    from litellm.litellm_core_utils.redact_messages import redacted_standard_logging_payload
 
-                    response_copy: Final = deepcopy(response)
-                    # Redact content in output array
-                    if isinstance(response_copy.get("output"), list):
-                        for output_item in response_copy["output"]:
-                            if isinstance(output_item, dict) and "content" in output_item:
-                                if isinstance(output_item["content"], list):
-                                    # Redact text in content items
-                                    for content_item in output_item["content"]:
-                                        if isinstance(content_item, dict) and "text" in content_item:
-                                            content_item["text"] = redacted_str
-                    standard_logging_object_copy["response"] = response_copy
+                    standard_logging_object_copy["response"] = redacted_standard_logging_payload(
+                        {"response": response}
+                    )["response"]
                 else:
                     # Standard ModelResponse format
                     model_response: Final = ModelResponse(choices=[Choices(message=Message(content=redacted_str))])

litellm/litellm_core_utils/redact_messages.py

@@ -145,31 +145,53 @@ def _redact_responses_api_output(output_items):
             output_item.input = REDACTED_BY_LITELLM
 
 
-def _redact_responses_api_output_dict(output_items, redacted_str: str):
-    """Helper to redact ResponsesAPIResponse output items in dict form."""
-    for output_item in output_items:
-        if not isinstance(output_item, dict):
-            continue
+def _redacted_responses_api_output_part(part: object, fields: tuple[str, ...], redacted_str: str) -> object:
+    if not isinstance(part, dict):
+        return part
+    return {key: redacted_str if key in fields and value is not None else value for key, value in part.items()}
 
-        if output_item.get("text") is not None:
-            output_item["text"] = redacted_str
 
-        if isinstance(output_item.get("content"), list):
-            for content_item in output_item["content"]:
-                if isinstance(content_item, dict) and content_item.get("text") is not None:
-                    content_item["text"] = redacted_str
-                if isinstance(content_item, dict) and content_item.get("refusal") is not None:
-                    content_item["refusal"] = redacted_str
+def _redacted_responses_api_output_item(item: object, redacted_str: str) -> object:
+    if not isinstance(item, dict):
+        return item
+    content: Final = item.get("content")
+    summary: Final = item.get("summary")
+    return {
+        **item,
+        **({"text": redacted_str} if item.get("text") is not None else {}),
+        **(
+            {
+                "content": [
+                    _redacted_responses_api_output_part(part, ("text", "refusal"), redacted_str) for part in content
+                ]
+            }
+            if isinstance(content, list)
+            else {}
+        ),
+        **(
+            {"summary": [_redacted_responses_api_output_part(part, ("text",), redacted_str) for part in summary]}
+            if item.get("type") == "reasoning" and isinstance(summary, list)
+            else {}
+        ),
+        **({"arguments": redacted_str} if item.get("type") == "function_call" and "arguments" in item else {}),
+        **({"input": redacted_str} if item.get("type") == "custom_tool_call" and "input" in item else {}),
+    }
 
-        if output_item.get("type") == "reasoning" and isinstance(output_item.get("summary"), list):
-            for summary_item in output_item["summary"]:
-                if isinstance(summary_item, dict) and summary_item.get("text") is not None:
-                    summary_item["text"] = redacted_str
 
-        if output_item.get("type") == "function_call" and "arguments" in output_item:
-            output_item["arguments"] = redacted_str
-        if output_item.get("type") == "custom_tool_call" and "input" in output_item:
-            output_item["input"] = redacted_str
+def _redact_responses_api_output_dict(output_items: list[object], redacted_str: str) -> list[object]:
+    return [_redacted_responses_api_output_item(item, redacted_str) for item in output_items]
+
+
+def _redacted_responses_api_response(response: Mapping[str, object]) -> dict[str, object]:
+    output: Final = response.get("output")
+    return {
+        **response,
+        "output": _redact_responses_api_output_dict(output, REDACTED_BY_LITELLM)
+        if isinstance(output, list)
+        else output,
+        **({"instructions": REDACTED_BY_LITELLM} if response.get("instructions") is not None else {}),
+        **({"reasoning": None} if response.get("reasoning") is not None else {}),
+    }
 
 
 def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[str, object]:
@@ -193,10 +215,8 @@ def _redact_standard_logging_object(payload: Mapping[str, object]) -> dict[str,
     response: Final = standard_logging_object.get("response")
     if response is not None:
         if isinstance(response, dict) and "output" in response:
-            # ResponsesAPIResponse format - redact content in output items
-            if isinstance(response.get("output"), list):
-                _redact_responses_api_output_dict(response["output"], redacted_str)
-            redact_vertex_ai_metadata_from_logged_object(response)
+            standard_logging_object["response"] = _redacted_responses_api_response(response)
+            redact_vertex_ai_metadata_from_logged_object(standard_logging_object["response"])
         elif isinstance(response, dict) and "choices" in response:
             # ModelResponse dict format - redact content in choices
             if isinstance(response.get("choices"), list):
@@ -309,14 +329,16 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
                 _redact_model_response_dict_choices(_result["choices"], REDACTED_BY_LITELLM)
             redact_vertex_ai_metadata_from_logged_object(_result)
         elif isinstance(_result, dict) and "output" in _result:
-            if isinstance(_result.get("output"), list):
-                _redact_responses_api_output_dict(_result["output"], REDACTED_BY_LITELLM)
+            return _redacted_responses_api_response(_result)
         elif isinstance(_result, litellm.ResponsesAPIResponse):
             if hasattr(_result, "output"):
                 _redact_responses_api_output(_result.output)
             # Redact reasoning field in ResponsesAPIResponse
             if hasattr(_result, "reasoning") and _result.reasoning is not None:
                 _result.reasoning = None
+            return _result.model_copy(
+                update={"instructions": REDACTED_BY_LITELLM} if _result.instructions is not None else {}
+            )
         elif isinstance(_result, litellm.EmbeddingResponse):
             if hasattr(_result, "data") and _result.data is not None:
                 _result.data = []

tests/unit/litellm_core_utils/test_redact_messages.py

@@ -6,6 +6,8 @@ but litellm_params["litellm_metadata"] is None.
 """
 
 import threading
+import copy
+import json
 from typing import Final
 from types import SimpleNamespace
 
@@ -15,6 +17,7 @@ import litellm
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.redact_messages import (
     _redact_responses_api_output,
+    _redacted_responses_api_response,
     perform_redaction,
     redact_streaming_responses_for_custom_logger,
     redacted_standard_logging_payload,
@@ -23,6 +26,59 @@ from litellm.litellm_core_utils.redact_messages import (
 from litellm.responses.main import mock_responses_api_response
 
 
+@pytest.mark.parametrize("surface", ("typed", "dict", "standard", "callback", "helper"))
+def test_responses_redaction_removes_instructions_without_changing_the_response(surface: str) -> None:
+    response: Final = litellm.ResponsesAPIResponse.model_validate(
+        {
+            **mock_responses_api_response("private answer").model_dump(),
+            "instructions": "private system instructions",
+            "reasoning": {"effort": "low", "summary": "auto"},
+            "output": [
+                {
+                    "type": "reasoning",
+                    "id": "rs_test",
+                    "summary": [{"type": "summary_text", "text": "private reasoning"}],
+                },
+                {
+                    "type": "function_call",
+                    "id": "fc_test",
+                    "call_id": "call_test",
+                    "name": "lookup",
+                    "arguments": "private arguments",
+                },
+            ],
+        }
+    )
+    original: Final = response.model_dump()
+    payload: Final = {"response": copy.deepcopy(original), "model": "test-model"}
+    logger: Final = CustomLogger()
+    logger.turn_off_message_logging = True
+
+    surfaces: Final = {
+        "typed": lambda: perform_redaction({}, response).model_dump(),
+        "dict": lambda: perform_redaction({}, original),
+        "helper": lambda: _redacted_responses_api_response(original),
+        "standard": lambda: redacted_standard_logging_payload(payload)["response"],
+        "callback": lambda: logger.redact_standard_logging_payload_from_model_call_details(
+            {"standard_logging_object": payload}
+        )["standard_logging_object"]["response"],
+    }
+    redacted: Final = surfaces[surface]()
+
+    assert redacted == {
+        **original,
+        "instructions": "redacted-by-litellm",
+        "reasoning": None,
+        "output": [
+            {**original["output"][0], "summary": [{"type": "summary_text", "text": "redacted-by-litellm"}]},
+            {**original["output"][1], "arguments": "redacted-by-litellm"},
+        ],
+    }
+    assert "private" not in json.dumps(redacted)
+    assert response.model_dump() == original
+    assert payload["response"] == original
+
+
 @pytest.fixture(autouse=True)
 def _reset_global_redaction():
     """Ensure the global setting is off for every test."""
@@ -197,7 +253,7 @@ class TestPerformRedaction:
         result = {
             "output": [
                 {"text": "top-level result"},
-                {"content": [{"text": "nested result"}]},
+                {"content": [{"text": "nested result"}, "non-dict content item"]},
                 {"type": "reasoning", "summary": [{"text": "reasoning result"}]},
             ],
             "usage": {"total_tokens": 1},
@@ -224,6 +280,7 @@ class TestPerformRedaction:
         assert redacted["usage"] == {"total_tokens": 1}
         assert redacted["output"][0]["text"] == "redacted-by-litellm"
         assert redacted["output"][1]["content"][0]["text"] == "redacted-by-litellm"
+        assert redacted["output"][1]["content"][1] == "non-dict content item"
         assert redacted["output"][2]["summary"][0]["text"] == "redacted-by-litellm"
         assert result["output"][0]["text"] == "top-level result"
 
@@ -661,12 +718,14 @@ class TestPerformRedaction:
         none_dict = {"type": "output_text", "text": None, "content": [{"text": None}]}
         real_dict = {"type": "output_text", "text": "real answer", "content": [{"text": "real part"}]}
 
-        _redact_responses_api_output_dict([none_dict, real_dict], "redacted-by-litellm")
+        redacted: Final = _redact_responses_api_output_dict([none_dict, real_dict], "redacted-by-litellm")
 
-        assert none_dict["text"] is None
-        assert none_dict["content"][0]["text"] is None
-        assert real_dict["text"] == "redacted-by-litellm"
-        assert real_dict["content"][0]["text"] == "redacted-by-litellm"
+        assert redacted == [
+            {"type": "output_text", "text": None, "content": [{"text": None}]},
+            {"type": "output_text", "text": "redacted-by-litellm", "content": [{"text": "redacted-by-litellm"}]},
+        ]
+        assert none_dict == {"type": "output_text", "text": None, "content": [{"text": None}]}
+        assert real_dict == {"type": "output_text", "text": "real answer", "content": [{"text": "real part"}]}
 
     def test_skips_non_dict_response_output_items(self):
         result = {