diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 340bcf32c3b..0b028ccec4a 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -447,22 +447,22 @@ def _redacted_responses_api_response( _REQUEST_TEXT_KEYS: Final = ("messages", "input", "prompt", "system", "instructions") _STRUCTURAL_KEYS: Final = frozenset({"role", "type", "id", "tool_call_id", "name"}) -_MIN_SCRUBBED_TEXT_LENGTH: Final = 4 +_MIN_UNQUOTED_TEXT_LENGTH: Final = 4 _ECHO_FRAGMENT_LENGTH: Final = 24 _MAX_SCRUB_COMPARISONS: Final = 10_000_000 def request_texts(*values: object) -> frozenset[str]: """ - The strings a request carries under ``values``, skipping structural fields such as ``role`` and - ``type`` and strings too short to point at the prompt. + The non-empty strings a request carries under ``values``, skipping structural fields such as ``role`` + and ``type``. """ texts: Final[set[str]] = set() # mutable-ok: filled by the walk below pending: Final[list[object]] = list(values) # mutable-ok: the walk's work stack while pending: value = pending.pop() if isinstance(value, str): - if len(value) >= _MIN_SCRUBBED_TEXT_LENGTH: + if value: texts.add(value) elif (mapping := _as_mapping(value)) is not None: pending.extend(item for key, item in mapping.items() if key not in _STRUCTURAL_KEYS) @@ -508,7 +508,13 @@ def scrub_request_text(text: str, texts: Collection[str]) -> str: quoted: Final = sorted((form for form in forms if form in text), key=len, reverse=True) if not quoted: return text - return re.sub("|".join(map(re.escape, quoted)), REDACTED_BY_LITELLM, text) + return re.sub("|".join(map(_echo_pattern, quoted)), REDACTED_BY_LITELLM, text) + + +def _echo_pattern(form: str) -> str: + """A quote of ``form``. A form short enough to occur in ordinary wording only counts between quotes.""" + escaped: Final = re.escape(form) + return escaped if len(form) >= _MIN_UNQUOTED_TEXT_LENGTH else f"(?<=['\"]){escaped}(?=['\"])" def scrubbed_error_information( @@ -553,16 +559,19 @@ def scrubbed_error_fields(payload: Mapping[str, object], texts: Collection[str]) def scrubbed_exception_text(model_call_details: Mapping[str, object]) -> tuple[str, str] | None: """ The error message and traceback a failure payload logged when redaction scrubbed the request's text - out of them, for callbacks that would otherwise record the raw exception. ``None`` when nothing was + out of either, for callbacks that would otherwise record the raw exception. ``None`` when nothing was scrubbed. """ payload: Final = _as_mapping(model_call_details.get("standard_logging_object")) or {} error_information: Final = _as_mapping(payload.get("error_information")) or {} message: Final = error_information.get("error_message") - if not isinstance(message, str) or REDACTED_BY_LITELLM not in message: - return None traceback_text: Final = error_information.get("traceback") - return message, traceback_text if isinstance(traceback_text, str) else "" + if not any(isinstance(text, str) and REDACTED_BY_LITELLM in text for text in (message, traceback_text)): + return None + return ( + message if isinstance(message, str) else "", + traceback_text if isinstance(traceback_text, str) else "", + ) def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[str, object]: @@ -851,26 +860,27 @@ def should_redact_message_logging(model_call_details: dict) -> bool: def should_redact_failure_logging(model_call_details: Mapping[str, object]) -> bool: """ ``should_redact_message_logging`` for the failure path, which must log the failure whatever the - request carried. Headers that are not a mapping make the payload build fail on the success and - failure paths alike, so the failure is logged unredacted, as before. + request carried. Headers that are not a mapping carry no redaction header, so the request's own + setting or the global one decides. """ try: return should_redact_message_logging(dict(model_call_details)) except AttributeError: - return False + dynamic_turn_off: Final = _get_turn_off_message_logging_from_dynamic_params(dict(model_call_details)) + return dynamic_turn_off if dynamic_turn_off is not None else litellm.turn_off_message_logging is True def should_redact_failed_request(request_data: Mapping[str, object]) -> bool: """ The redaction decision for proxy failure hooks, which get the request body but not its logging - object: the same metadata, headers and dynamic params the logging object would read. Callback - params the logging object rejects leave the failure unredacted, as before. + object: the same metadata, headers and dynamic params the logging object would read. When the + logging object would reject the request's callback params, the global setting decides. """ body: Final = dict(request_data) try: dynamic_params: Final = initialize_standard_callback_dynamic_params(body) except ValueError: - return False + return litellm.turn_off_message_logging is True return should_redact_failure_logging( { "litellm_params": {key: body[key] for key in ("metadata", "litellm_metadata") if key in body}, diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index b4378e12ec1..f40d11d3b23 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -11782,9 +11782,10 @@ async def test_failure_with_non_mapping_headers_is_still_logged(monkeypatch, hea monkeypatch.setattr(litellm, "_async_failure_callback", [capture]) logging_obj: Final = _prompt_failure_logging_obj({"metadata": {"headers": headers}}) - await logging_obj.async_failure_handler(_prompt_echo_error(), "Traceback") + await logging_obj.async_failure_handler(_prompt_echo_error(), f"Traceback: {SCRUB_PROMPT}") assert capture.async_kwargs is not None + assert (SCRUB_PROMPT in capture.async_kwargs["traceback_exception"]) is not redact def test_sync_failure_hands_callbacks_and_langfuse_the_scrubbed_error(monkeypatch): diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index 8eedd755e60..a555a5c73e3 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -36,6 +36,7 @@ from litellm.litellm_core_utils.redact_messages import ( request_texts, scrub_request_text, scrubbed_error_information_for_request, + scrubbed_exception_text, should_redact_failed_request, should_redact_message_logging, ) @@ -2102,6 +2103,12 @@ def test_scrub_request_text_leaves_an_error_that_does_not_quote_the_prompt() -> assert scrub_request_text("Incorrect API key provided", request_texts(PROMPT, "hi")) == "Incorrect API key provided" +def test_scrub_request_text_replaces_a_short_text_only_between_quotes() -> None: + scrubbed: Final = scrub_request_text("Invalid value '432' and \"432\"; 432 is too short", request_texts("432")) + + assert scrubbed == "Invalid value 'redacted-by-litellm' and \"redacted-by-litellm\"; 432 is too short" + + def test_scrub_request_text_replaces_the_whole_text_when_the_search_is_too_large() -> None: texts: Final = request_texts([f"message number {index}" for index in range(5_000)]) @@ -2139,12 +2146,36 @@ def test_should_redact_failed_request_follows_the_request_opt_ins(request_data: [{"metadata": {"headers": "x"}}, {"turn_off_message_logging": "os.environ/REDACT"}], ids=["non_mapping_headers", "env_reference"], ) -def test_should_redact_failed_request_leaves_an_undecidable_request_unredacted( - monkeypatch: pytest.MonkeyPatch, request_data: dict +@pytest.mark.parametrize("global_redaction", [True, False], ids=["global_on", "global_off"]) +def test_should_redact_failed_request_falls_back_to_the_global_setting_for_an_undecidable_request( + monkeypatch: pytest.MonkeyPatch, request_data: dict, global_redaction: bool ) -> None: - monkeypatch.setattr(litellm, "turn_off_message_logging", True) + monkeypatch.setattr(litellm, "turn_off_message_logging", global_redaction) - assert should_redact_failed_request(request_data) is False + assert should_redact_failed_request(request_data) is global_redaction + + +def test_should_redact_failed_request_reads_the_request_setting_past_non_mapping_headers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + + assert should_redact_failed_request({"metadata": {"headers": "x"}, "turn_off_message_logging": True}) is True + + +@pytest.mark.parametrize( + "error_information", + [ + {"error_message": "bad: redacted-by-litellm", "traceback": "tb"}, + {"error_message": "bad input", "traceback": "tb redacted-by-litellm"}, + ], + ids=["message", "traceback_only"], +) +def test_scrubbed_exception_text_reports_a_scrub_of_either_field(error_information: dict) -> None: + logged: Final = scrubbed_exception_text({"standard_logging_object": {"error_information": error_information}}) + + assert logged == (error_information["error_message"], error_information["traceback"]) + assert scrubbed_exception_text({"standard_logging_object": {"error_information": {"error_message": "bad"}}}) is None def test_scrubbed_error_information_for_request_scrubs_only_when_the_request_redacts() -> None: