diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index d4162369a35..0d281961774 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -940,20 +940,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac response: Final = standard_logging_object_copy["response"] # Check if this is a ResponsesAPIResponse (has "output" field) if isinstance(response, dict) and "output" in response: - # Make a copy to avoid modifying the original - from copy import deepcopy + from litellm.litellm_core_utils.redact_messages import redacted_standard_logging_payload - response_copy: Final = deepcopy(response) - # Redact content in output array - if isinstance(response_copy.get("output"), list): - for output_item in response_copy["output"]: - if isinstance(output_item, dict) and "content" in output_item: - if isinstance(output_item["content"], list): - # Redact text in content items - for content_item in output_item["content"]: - if isinstance(content_item, dict) and "text" in content_item: - content_item["text"] = redacted_str - standard_logging_object_copy["response"] = response_copy + standard_logging_object_copy["response"] = redacted_standard_logging_payload( + {"response": response} + )["response"] else: # Standard ModelResponse format model_response: Final = ModelResponse(choices=[Choices(message=Message(content=redacted_str))]) diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 8c77ef32cfb..ad18747c869 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -145,31 +145,53 @@ def _redact_responses_api_output(output_items): output_item.input = REDACTED_BY_LITELLM -def _redact_responses_api_output_dict(output_items, redacted_str: str): - """Helper to redact ResponsesAPIResponse output items in dict form.""" - for output_item in output_items: - if not isinstance(output_item, dict): - continue +def _redacted_responses_api_output_part(part: object, fields: tuple[str, ...], redacted_str: str) -> object: + if not isinstance(part, dict): + return part + return {key: redacted_str if key in fields and value is not None else value for key, value in part.items()} - if output_item.get("text") is not None: - output_item["text"] = redacted_str - if isinstance(output_item.get("content"), list): - for content_item in output_item["content"]: - if isinstance(content_item, dict) and content_item.get("text") is not None: - content_item["text"] = redacted_str - if isinstance(content_item, dict) and content_item.get("refusal") is not None: - content_item["refusal"] = redacted_str +def _redacted_responses_api_output_item(item: object, redacted_str: str) -> object: + if not isinstance(item, dict): + return item + content: Final = item.get("content") + summary: Final = item.get("summary") + return { + **item, + **({"text": redacted_str} if item.get("text") is not None else {}), + **( + { + "content": [ + _redacted_responses_api_output_part(part, ("text", "refusal"), redacted_str) for part in content + ] + } + if isinstance(content, list) + else {} + ), + **( + {"summary": [_redacted_responses_api_output_part(part, ("text",), redacted_str) for part in summary]} + if item.get("type") == "reasoning" and isinstance(summary, list) + else {} + ), + **({"arguments": redacted_str} if item.get("type") == "function_call" and "arguments" in item else {}), + **({"input": redacted_str} if item.get("type") == "custom_tool_call" and "input" in item else {}), + } - if output_item.get("type") == "reasoning" and isinstance(output_item.get("summary"), list): - for summary_item in output_item["summary"]: - if isinstance(summary_item, dict) and summary_item.get("text") is not None: - summary_item["text"] = redacted_str - if output_item.get("type") == "function_call" and "arguments" in output_item: - output_item["arguments"] = redacted_str - if output_item.get("type") == "custom_tool_call" and "input" in output_item: - output_item["input"] = redacted_str +def _redact_responses_api_output_dict(output_items: list[object], redacted_str: str) -> list[object]: + return [_redacted_responses_api_output_item(item, redacted_str) for item in output_items] + + +def _redacted_responses_api_response(response: Mapping[str, object]) -> dict[str, object]: + output: Final = response.get("output") + return { + **response, + "output": _redact_responses_api_output_dict(output, REDACTED_BY_LITELLM) + if isinstance(output, list) + else output, + **({"instructions": REDACTED_BY_LITELLM} if response.get("instructions") is not None else {}), + **({"reasoning": None} if response.get("reasoning") is not None else {}), + } def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[str, object]: @@ -193,10 +215,8 @@ def _redact_standard_logging_object(payload: Mapping[str, object]) -> dict[str, response: Final = standard_logging_object.get("response") if response is not None: if isinstance(response, dict) and "output" in response: - # ResponsesAPIResponse format - redact content in output items - if isinstance(response.get("output"), list): - _redact_responses_api_output_dict(response["output"], redacted_str) - redact_vertex_ai_metadata_from_logged_object(response) + standard_logging_object["response"] = _redacted_responses_api_response(response) + redact_vertex_ai_metadata_from_logged_object(standard_logging_object["response"]) elif isinstance(response, dict) and "choices" in response: # ModelResponse dict format - redact content in choices if isinstance(response.get("choices"), list): @@ -309,14 +329,16 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons _redact_model_response_dict_choices(_result["choices"], REDACTED_BY_LITELLM) redact_vertex_ai_metadata_from_logged_object(_result) elif isinstance(_result, dict) and "output" in _result: - if isinstance(_result.get("output"), list): - _redact_responses_api_output_dict(_result["output"], REDACTED_BY_LITELLM) + return _redacted_responses_api_response(_result) elif isinstance(_result, litellm.ResponsesAPIResponse): if hasattr(_result, "output"): _redact_responses_api_output(_result.output) # Redact reasoning field in ResponsesAPIResponse if hasattr(_result, "reasoning") and _result.reasoning is not None: _result.reasoning = None + return _result.model_copy( + update={"instructions": REDACTED_BY_LITELLM} if _result.instructions is not None else {} + ) elif isinstance(_result, litellm.EmbeddingResponse): if hasattr(_result, "data") and _result.data is not None: _result.data = [] diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 76d037ce760..52d54ccd3cc 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -6,6 +6,8 @@ but litellm_params["litellm_metadata"] is None. """ import threading +import copy +import json from typing import Final from types import SimpleNamespace @@ -15,6 +17,7 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.redact_messages import ( _redact_responses_api_output, + _redacted_responses_api_response, perform_redaction, redact_streaming_responses_for_custom_logger, redacted_standard_logging_payload, @@ -23,6 +26,59 @@ from litellm.litellm_core_utils.redact_messages import ( from litellm.responses.main import mock_responses_api_response +@pytest.mark.parametrize("surface", ("typed", "dict", "standard", "callback", "helper")) +def test_responses_redaction_removes_instructions_without_changing_the_response(surface: str) -> None: + response: Final = litellm.ResponsesAPIResponse.model_validate( + { + **mock_responses_api_response("private answer").model_dump(), + "instructions": "private system instructions", + "reasoning": {"effort": "low", "summary": "auto"}, + "output": [ + { + "type": "reasoning", + "id": "rs_test", + "summary": [{"type": "summary_text", "text": "private reasoning"}], + }, + { + "type": "function_call", + "id": "fc_test", + "call_id": "call_test", + "name": "lookup", + "arguments": "private arguments", + }, + ], + } + ) + original: Final = response.model_dump() + payload: Final = {"response": copy.deepcopy(original), "model": "test-model"} + logger: Final = CustomLogger() + logger.turn_off_message_logging = True + + surfaces: Final = { + "typed": lambda: perform_redaction({}, response).model_dump(), + "dict": lambda: perform_redaction({}, original), + "helper": lambda: _redacted_responses_api_response(original), + "standard": lambda: redacted_standard_logging_payload(payload)["response"], + "callback": lambda: logger.redact_standard_logging_payload_from_model_call_details( + {"standard_logging_object": payload} + )["standard_logging_object"]["response"], + } + redacted: Final = surfaces[surface]() + + assert redacted == { + **original, + "instructions": "redacted-by-litellm", + "reasoning": None, + "output": [ + {**original["output"][0], "summary": [{"type": "summary_text", "text": "redacted-by-litellm"}]}, + {**original["output"][1], "arguments": "redacted-by-litellm"}, + ], + } + assert "private" not in json.dumps(redacted) + assert response.model_dump() == original + assert payload["response"] == original + + @pytest.fixture(autouse=True) def _reset_global_redaction(): """Ensure the global setting is off for every test.""" @@ -197,7 +253,7 @@ class TestPerformRedaction: result = { "output": [ {"text": "top-level result"}, - {"content": [{"text": "nested result"}]}, + {"content": [{"text": "nested result"}, "non-dict content item"]}, {"type": "reasoning", "summary": [{"text": "reasoning result"}]}, ], "usage": {"total_tokens": 1}, @@ -224,6 +280,7 @@ class TestPerformRedaction: assert redacted["usage"] == {"total_tokens": 1} assert redacted["output"][0]["text"] == "redacted-by-litellm" assert redacted["output"][1]["content"][0]["text"] == "redacted-by-litellm" + assert redacted["output"][1]["content"][1] == "non-dict content item" assert redacted["output"][2]["summary"][0]["text"] == "redacted-by-litellm" assert result["output"][0]["text"] == "top-level result" @@ -661,12 +718,14 @@ class TestPerformRedaction: none_dict = {"type": "output_text", "text": None, "content": [{"text": None}]} real_dict = {"type": "output_text", "text": "real answer", "content": [{"text": "real part"}]} - _redact_responses_api_output_dict([none_dict, real_dict], "redacted-by-litellm") + redacted: Final = _redact_responses_api_output_dict([none_dict, real_dict], "redacted-by-litellm") - assert none_dict["text"] is None - assert none_dict["content"][0]["text"] is None - assert real_dict["text"] == "redacted-by-litellm" - assert real_dict["content"][0]["text"] == "redacted-by-litellm" + assert redacted == [ + {"type": "output_text", "text": None, "content": [{"text": None}]}, + {"type": "output_text", "text": "redacted-by-litellm", "content": [{"text": "redacted-by-litellm"}]}, + ] + assert none_dict == {"type": "output_text", "text": None, "content": [{"text": None}]} + assert real_dict == {"type": "output_text", "text": "real answer", "content": [{"text": "real part"}]} def test_skips_non_dict_response_output_items(self): result = {