diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 49cac4fbc0c..a1b2615e81b 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -9,6 +9,7 @@ import asyncio import copy +import functools import inspect import itertools import json @@ -448,7 +449,10 @@ def _redacted_responses_api_response( _REQUEST_TEXT_KEYS: Final = ("messages", "input", "prompt", "system", "instructions") _STRUCTURAL_KEYS: Final = frozenset({"role", "type", "id", "tool_call_id", "name"}) _MIN_UNQUOTED_TEXT_LENGTH: Final = 4 -_ECHO_FRAGMENT_LENGTH: Final = 24 +# pydantic shows a long input value as its repr's first 25 and last 24 characters around "...", quotes and +# container keys included, so each side of the cut shows at most 24 characters of the text +_MIN_CUT_ECHO_LENGTH: Final = 12 +_MAX_CUT_ECHO_LENGTH: Final = 24 _MAX_SCRUB_COMPARISONS: Final = 10_000_000 @@ -485,15 +489,23 @@ def failure_request_texts(model_call_details: Mapping[str, object], *originals: ) -def _echo_forms(text: str) -> tuple[str, ...]: - """How an error can quote ``text``: verbatim or JSON or repr escaped, whole or cut down to its head and tail.""" - whole: Final = {text, json.dumps(text)[1:-1], json.dumps(text, ensure_ascii=False)[1:-1], repr(text)[1:-1]} - long_forms: Final = tuple(form for form in whole if len(form) >= 2 * _ECHO_FRAGMENT_LENGTH) - return ( - *whole, - *(form[:_ECHO_FRAGMENT_LENGTH] for form in long_forms), - *(form[-_ECHO_FRAGMENT_LENGTH:] for form in long_forms), - ) +def _echo_forms(text: str) -> frozenset[str]: + """How an error can quote ``text``: verbatim, or JSON or repr escaped.""" + return frozenset({text, json.dumps(text)[1:-1], json.dumps(text, ensure_ascii=False)[1:-1], repr(text)[1:-1]}) + + +def _cut_head_pattern(form: str) -> str: + """The start of ``form`` where an error cuts it short with ``...``, as pydantic shows a long input value.""" + rest: Final = form[_MIN_CUT_ECHO_LENGTH:_MAX_CUT_ECHO_LENGTH] + optional_rest: Final = functools.reduce(lambda inner, char: f"(?:{re.escape(char)}{inner})?", reversed(rest), "") + return f"{re.escape(form[:_MIN_CUT_ECHO_LENGTH])}{optional_rest}(?=\\.\\.\\.)" + + +def _cut_tail_pattern(form: str) -> str: + """The end of ``form`` where an error picks it up again after ``...``.""" + rest: Final = form[-_MAX_CUT_ECHO_LENGTH:-_MIN_CUT_ECHO_LENGTH] + optional_rest: Final = functools.reduce(lambda inner, char: f"(?:{inner}{re.escape(char)})?", rest, "") + return f"(?<=\\.\\.\\.){optional_rest}{re.escape(form[-_MIN_CUT_ECHO_LENGTH:])}" def scrub_request_text(text: str, texts: Collection[str]) -> str: @@ -508,9 +520,14 @@ def scrub_request_text(text: str, texts: Collection[str]) -> str: return REDACTED_BY_LITELLM forms: Final = frozenset(itertools.chain.from_iterable(map(_echo_forms, texts))) quoted: Final = sorted((form for form in forms if form in text), key=len, reverse=True) - if not quoted: + cut_forms: Final = tuple(form for form in forms if len(form) > _MIN_CUT_ECHO_LENGTH) if "..." in text else () + cut: Final = ( + *(_cut_head_pattern(form) for form in cut_forms if form[:_MIN_CUT_ECHO_LENGTH] in text), + *(_cut_tail_pattern(form) for form in cut_forms if form[-_MIN_CUT_ECHO_LENGTH:] in text), + ) + if not quoted and not cut: return text - return re.sub("|".join(map(_echo_pattern, quoted)), REDACTED_BY_LITELLM, text) + return re.sub("|".join((*map(_echo_pattern, quoted), *cut)), REDACTED_BY_LITELLM, text) def _echo_pattern(form: str) -> str: @@ -559,14 +576,16 @@ def scrubbed_error_fields(payload: Mapping[str, object], texts: Collection[str]) def scrubbed_exception_text(model_call_details: Mapping[str, object]) -> tuple[str, str] | None: """ - The error message and traceback a failure payload logged when redaction scrubbed the request's text - out of either, for callbacks that would otherwise record the raw exception. ``None`` when nothing was - scrubbed. + The error message and traceback a failure logged when redaction scrubbed the request's text out of + either, for callbacks that would otherwise record the raw exception. ``None`` when nothing was scrubbed. """ payload: Final = _as_mapping(model_call_details.get("standard_logging_object")) or {} error_information: Final = _as_mapping(payload.get("error_information")) or {} message: Final = error_information.get("error_message") - traceback_text: Final = error_information.get("traceback") + logged_traceback: Final = model_call_details.get("traceback_exception") + traceback_text: Final = ( + logged_traceback if isinstance(logged_traceback, str) else error_information.get("traceback") + ) if not any(isinstance(text, str) and REDACTED_BY_LITELLM in text for text in (message, traceback_text)): return None return ( diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 8bf7b9cf709..13eba882131 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -11806,6 +11806,20 @@ def test_sync_failure_hands_callbacks_and_langfuse_the_scrubbed_error(monkeypatc assert SCRUB_PROMPT not in status_message +def test_sync_failure_hands_lunary_the_scrubbed_traceback_when_only_the_traceback_quotes_the_prompt(monkeypatch): + lunary: Final = MagicMock() + monkeypatch.setattr(litellm, "turn_off_message_logging", True) + monkeypatch.setattr(litellm, "failure_callback", ["lunary"]) + monkeypatch.setattr(litellm_logging, "lunaryLogger", lunary) + error: Final = litellm.APIConnectionError(message="Connection failed", model="gpt-4o", llm_provider="openai") + + _prompt_failure_logging_obj().failure_handler(error, f"ValueError: bad input {SCRUB_PROMPT}\n\n{error}") + + logged_error: Final = lunary.log_event.call_args.kwargs["error"] + assert "ValueError: bad input redacted-by-litellm" in logged_error + assert SCRUB_PROMPT not in logged_error + + def test_enterprise_alerting_loggers_resolves_real_classes(): from litellm_enterprise.enterprise_callbacks.pagerduty.pagerduty import PagerDutyAlerting from litellm_enterprise.enterprise_callbacks.send_emails.resend_email import ResendEmailLogger diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index a555a5c73e3..38dd42ca442 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -20,7 +20,7 @@ from unittest.mock import patch import httpx import pytest import pytest_asyncio -from pydantic import JsonValue +from pydantic import JsonValue, TypeAdapter, ValidationError import litellm from litellm.constants import LOGGING_WORKER_MAX_TIME_PER_COROUTINE @@ -2099,6 +2099,17 @@ def test_scrub_request_text_replaces_each_way_an_error_quotes_the_prompt(quoted: assert scrubbed.startswith("Invalid content: <") and scrubbed.endswith("> rejected") +@pytest.mark.parametrize("value", [PROMPT, [PROMPT], {"content": PROMPT}], ids=["str", "list", "dict"]) +def test_scrub_request_text_replaces_both_sides_of_a_pydantic_cut_input_value(value: object) -> None: + with pytest.raises(ValidationError, match="input_value=") as raised: + TypeAdapter(int).validate_python(value) + + scrubbed: Final = scrub_request_text(str(raised.value), request_texts(PROMPT)) + + assert "my ssn" not in scrubbed and "4321" not in scrubbed + assert "redacted-by-litellm...redacted-by-litellm" in scrubbed + + def test_scrub_request_text_leaves_an_error_that_does_not_quote_the_prompt() -> None: assert scrub_request_text("Incorrect API key provided", request_texts(PROMPT, "hi")) == "Incorrect API key provided" @@ -2178,6 +2189,15 @@ def test_scrubbed_exception_text_reports_a_scrub_of_either_field(error_informati assert scrubbed_exception_text({"standard_logging_object": {"error_information": {"error_message": "bad"}}}) is None +def test_scrubbed_exception_text_reports_a_scrub_of_the_logged_traceback() -> None: + model_call_details: Final = { + "traceback_exception": "ValueError: bad: redacted-by-litellm", + "standard_logging_object": {"error_information": {"error_message": "outer", "traceback": "frames"}}, + } + + assert scrubbed_exception_text(model_call_details) == ("outer", "ValueError: bad: redacted-by-litellm") + + def test_scrubbed_error_information_for_request_scrubs_only_when_the_request_redacts() -> None: error_information: Final = StandardLoggingPayloadErrorInformation( error_code="400", error_class="BadRequestError", error_message=f"bad: {PROMPT}", traceback=f"tb {PROMPT}"