From ca6ddfe93e6a64ab64b3bdccde017efa31f122e7 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:11:19 +0000 Subject: [PATCH] fix(logging): redact secrets and escape control chars on set_verbose stdout paths --- .../enterprise_hooks/banned_keywords.py | 4 +- .../enterprise_hooks/blocked_user_list.py | 4 +- .../google_text_moderation.py | 4 +- .../enterprise_callbacks/llama_guard.py | 4 +- .../enterprise_callbacks/llm_guard.py | 4 +- litellm/_logging.py | 34 ++++++++- litellm/caching/caching.py | 4 +- .../litellm_core_utils/streaming_handler.py | 3 +- litellm/main.py | 4 +- .../guardrails/guardrail_hooks/presidio.py | 4 +- litellm/proxy/hooks/batch_redis_get.py | 4 +- .../proxy/hooks/parallel_request_limiter.py | 4 +- .../proxy/hooks/prompt_injection_detection.py | 4 +- litellm/proxy/utils.py | 4 +- litellm/utils.py | 4 +- tests/test_litellm/test_secret_redaction.py | 72 +++++++++++++++++++ 16 files changed, 133 insertions(+), 28 deletions(-) diff --git a/enterprise/enterprise_hooks/banned_keywords.py b/enterprise/enterprise_hooks/banned_keywords.py index 47421c96051..4a07667b436 100644 --- a/enterprise/enterprise_hooks/banned_keywords.py +++ b/enterprise/enterprise_hooks/banned_keywords.py @@ -16,7 +16,7 @@ from litellm.proxy.guardrails._content_utils import ( iter_message_text, ) from litellm.integrations.custom_logger import CustomLogger -from litellm._logging import verbose_proxy_logger +from litellm._logging import redact_and_sanitize, verbose_proxy_logger from fastapi import HTTPException @@ -54,7 +54,7 @@ class _ENTERPRISE_BannedKeywords(CustomLogger): verbose_proxy_logger.debug(print_statement) if litellm.set_verbose is True: - print(print_statement) # noqa + print(redact_and_sanitize(print_statement)) # noqa def test_violation(self, test_str: str): for word in self.banned_keywords_list: diff --git a/enterprise/enterprise_hooks/blocked_user_list.py b/enterprise/enterprise_hooks/blocked_user_list.py index d34605b30ac..56e9c6d1b58 100644 --- a/enterprise/enterprise_hooks/blocked_user_list.py +++ b/enterprise/enterprise_hooks/blocked_user_list.py @@ -13,7 +13,7 @@ from litellm.proxy.utils import PrismaClient from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth, LiteLLM_EndUserTable from litellm.integrations.custom_logger import CustomLogger -from litellm._logging import verbose_proxy_logger +from litellm._logging import redact_and_sanitize, verbose_proxy_logger from fastapi import HTTPException @@ -51,7 +51,7 @@ class _ENTERPRISE_BlockedUserList(CustomLogger): verbose_proxy_logger.debug(print_statement) if litellm.set_verbose is True: - print(print_statement) # noqa + print(redact_and_sanitize(print_statement)) # noqa async def async_pre_call_hook( self, diff --git a/enterprise/enterprise_hooks/google_text_moderation.py b/enterprise/enterprise_hooks/google_text_moderation.py index 5b2d71c5cca..a3387eb13c9 100644 --- a/enterprise/enterprise_hooks/google_text_moderation.py +++ b/enterprise/enterprise_hooks/google_text_moderation.py @@ -9,7 +9,7 @@ from fastapi import HTTPException import litellm -from litellm._logging import verbose_proxy_logger +from litellm._logging import redact_and_sanitize, verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.guardrails._content_utils import iter_message_text @@ -81,7 +81,7 @@ class _ENTERPRISE_GoogleTextModeration(CustomLogger): try: verbose_proxy_logger.debug(print_statement) if litellm.set_verbose: - print(print_statement) # noqa + print(redact_and_sanitize(print_statement)) # noqa except Exception: pass diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/llama_guard.py b/enterprise/litellm_enterprise/enterprise_callbacks/llama_guard.py index 5e1aebdbdfb..d78ec8a882b 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/llama_guard.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/llama_guard.py @@ -20,7 +20,7 @@ from typing import Literal, Optional from fastapi import HTTPException import litellm -from litellm._logging import verbose_proxy_logger +from litellm._logging import redact_and_sanitize, verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import CallTypesLiteral, Choices, ModelResponse @@ -55,7 +55,7 @@ class _ENTERPRISE_LlamaGuard(CustomLogger): try: verbose_proxy_logger.debug(print_statement) if litellm.set_verbose: - print(print_statement) # noqa + print(redact_and_sanitize(print_statement)) # noqa except Exception: pass diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py b/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py index d10b5a2ab09..2d7562c2c1e 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py @@ -14,7 +14,7 @@ import aiohttp from fastapi import HTTPException import litellm -from litellm._logging import verbose_proxy_logger +from litellm._logging import redact_and_sanitize, verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth from litellm.secret_managers.main import get_secret_str @@ -42,7 +42,7 @@ class _ENTERPRISE_LLMGuard(CustomLogger): try: verbose_proxy_logger.debug(print_statement) if litellm.set_verbose: - print(print_statement) # noqa + print(redact_and_sanitize(print_statement)) # noqa except Exception: pass diff --git a/litellm/_logging.py b/litellm/_logging.py index 5f3c483869d..fcce6cf4234 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -1,6 +1,7 @@ import ast import logging import os +import re import sys from datetime import datetime from logging import Formatter @@ -43,6 +44,37 @@ def redact_secrets(value: str) -> str: return _redact_string(value) +_CONTROL_CHAR_RE = re.compile(r"[\x00-\x08\x0a-\x1f\x7f-\x9f\u2028\u2029]") +_CONTROL_CHAR_NAMES = {"\n": "\\n", "\r": "\\r", "\x0b": "\\v", "\x0c": "\\f", "\x08": "\\b", "\x07": "\\a"} + + +def _escape_control_character(match: "re.Match[str]") -> str: + char = match.group() + named = _CONTROL_CHAR_NAMES.get(char) + if named is not None: + return named + codepoint = ord(char) + return f"\\x{codepoint:02x}" if codepoint < 0x100 else f"\\u{codepoint:04x}" + + +def sanitize_control_characters(value: str) -> str: + """Escape characters that let untrusted content forge log lines or drive the terminal. + + Covers C0/C1 controls (CR, LF, ESC, ...) and the Unicode line/paragraph + separators; tab is left alone since it cannot start a new log record. + """ + return _CONTROL_CHAR_RE.sub(_escape_control_character, value) + + +def redact_and_sanitize(print_statement: object) -> str: + """Render an arbitrary value for a stdout/print sink: secrets masked, control characters escaped. + + Use this for the `litellm.set_verbose` print paths, which bypass the + logging handlers and therefore SecretRedactionFilter. + """ + return sanitize_control_characters(redact_secrets(str(print_statement))) + + class SecretRedactionFilter(logging.Filter): """Scrubs known secret/credential patterns from log records.""" @@ -413,7 +445,7 @@ def _enable_debugging(): def print_verbose(print_statement): try: if set_verbose: - print(redact_secrets(str(print_statement))) # noqa: T201 + print(redact_and_sanitize(print_statement)) # noqa: T201 except Exception: pass diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 34badaa3e8a..2c670ea85e7 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -18,7 +18,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union from pydantic import BaseModel import litellm -from litellm._logging import verbose_logger +from litellm._logging import redact_and_sanitize, verbose_logger from litellm.constants import CACHED_STREAMING_CHUNK_DELAY from litellm.litellm_core_utils.model_param_helper import ModelParamHelper from litellm.types.caching import * @@ -41,7 +41,7 @@ def print_verbose(print_statement): try: verbose_logger.debug(print_statement) if litellm.set_verbose: - print(print_statement) # noqa: T201 + print(redact_and_sanitize(print_statement)) # noqa: T201 except Exception: pass diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 60dbf7c644a..5fa021b5b65 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -26,6 +26,7 @@ from pydantic import BaseModel import litellm from litellm import verbose_logger +from litellm._logging import redact_and_sanitize from litellm._uuid import uuid from litellm.litellm_core_utils.model_response_utils import ( is_model_response_stream_empty, @@ -93,7 +94,7 @@ def is_async_iterable(obj: Any) -> bool: def print_verbose(print_statement): try: if litellm.set_verbose: - print(print_statement) # noqa: T201 + print(redact_and_sanitize(print_statement)) # noqa: T201 except Exception: pass diff --git a/litellm/main.py b/litellm/main.py index dc3ec469a1b..41d4978e896 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -41,7 +41,7 @@ from typing import ( get_args, ) -from litellm._logging import _redact_string +from litellm._logging import _redact_string, redact_and_sanitize from litellm._uuid import uuid if TYPE_CHECKING: @@ -8381,7 +8381,7 @@ def print_verbose(print_statement): try: verbose_logger.debug(print_statement) if litellm.set_verbose: - print(print_statement) # noqa: T201 + print(redact_and_sanitize(print_statement)) # noqa: T201 except Exception: pass diff --git a/litellm/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/proxy/guardrails/guardrail_hooks/presidio.py index a0c822964a0..1fb4cb52e6e 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/proxy/guardrails/guardrail_hooks/presidio.py @@ -30,7 +30,7 @@ import aiohttp import litellm from litellm import get_secret -from litellm._logging import verbose_proxy_logger +from litellm._logging import redact_and_sanitize, verbose_proxy_logger from litellm.types.utils import GenericGuardrailAPIInputs if TYPE_CHECKING: @@ -1320,7 +1320,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail): try: verbose_proxy_logger.debug(print_statement) if litellm.set_verbose: - print(print_statement) # noqa: T201 + print(redact_and_sanitize(print_statement)) # noqa: T201 except Exception: pass diff --git a/litellm/proxy/hooks/batch_redis_get.py b/litellm/proxy/hooks/batch_redis_get.py index 8b11e185fb9..53a258e08c5 100644 --- a/litellm/proxy/hooks/batch_redis_get.py +++ b/litellm/proxy/hooks/batch_redis_get.py @@ -9,7 +9,7 @@ from typing import Literal, Optional from fastapi import HTTPException import litellm -from litellm._logging import verbose_proxy_logger +from litellm._logging import redact_and_sanitize, verbose_proxy_logger from litellm.caching.caching import DualCache, InMemoryCache, RedisCache from litellm.integrations.custom_logger import CustomLogger from litellm.proxy._types import UserAPIKeyAuth @@ -31,7 +31,7 @@ class _PROXY_BatchRedisRequests(CustomLogger): elif debug_level == "INFO": verbose_proxy_logger.debug(print_statement) if litellm.set_verbose is True: - print(print_statement) # noqa: T201 + print(redact_and_sanitize(print_statement)) # noqa: T201 async def async_pre_call_hook( self, diff --git a/litellm/proxy/hooks/parallel_request_limiter.py b/litellm/proxy/hooks/parallel_request_limiter.py index ee6abb13d6b..91987e62cfd 100644 --- a/litellm/proxy/hooks/parallel_request_limiter.py +++ b/litellm/proxy/hooks/parallel_request_limiter.py @@ -8,7 +8,7 @@ from typing_extensions import TypedDict import litellm from litellm import DualCache, EmbeddingResponse, ModelResponse, TextCompletionResponse -from litellm._logging import verbose_proxy_logger +from litellm._logging import redact_and_sanitize, verbose_proxy_logger from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs from litellm.proxy._types import CommonProxyErrors, CurrentItemRateLimit, UserAPIKeyAuth @@ -51,7 +51,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger): try: verbose_proxy_logger.debug(print_statement) if litellm.set_verbose: - print(print_statement) # noqa: T201 + print(redact_and_sanitize(print_statement)) # noqa: T201 except Exception: pass diff --git a/litellm/proxy/hooks/prompt_injection_detection.py b/litellm/proxy/hooks/prompt_injection_detection.py index 2d55a644bb2..4fd43acf376 100644 --- a/litellm/proxy/hooks/prompt_injection_detection.py +++ b/litellm/proxy/hooks/prompt_injection_detection.py @@ -13,7 +13,7 @@ from typing import List, Literal, Optional from fastapi import HTTPException import litellm -from litellm._logging import verbose_proxy_logger +from litellm._logging import redact_and_sanitize, verbose_proxy_logger from litellm.caching.caching import DualCache from litellm.constants import DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD from litellm.integrations.custom_logger import CustomLogger @@ -72,7 +72,7 @@ class _OPTIONAL_PromptInjectionDetection(CustomLogger): verbose_proxy_logger.debug(print_statement) if litellm.set_verbose is True: - print(print_statement) # noqa: T201 + print(redact_and_sanitize(print_statement)) # noqa: T201 def update_environment(self, router: Optional[Router] = None): self.llm_router = router diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index e85ccf150d2..78986d905c9 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -88,7 +88,7 @@ from litellm import ( ModelResponseStream, Router, ) -from litellm._logging import _redact_string, verbose_proxy_logger +from litellm._logging import _redact_string, redact_and_sanitize, verbose_proxy_logger from litellm._service_logger import ServiceLogging, ServiceTypes from litellm.caching.caching import DualCache, RedisCache from litellm.caching.dual_cache import LimitedSizeOrderedDict @@ -196,7 +196,7 @@ def print_verbose(print_statement): verbose_proxy_logger.debug("{}\n{}".format(print_statement, traceback.format_exc())) if litellm.set_verbose: - print(f"LiteLLM Proxy: {_redact_string(str(print_statement))}") # noqa: T201 + print(f"LiteLLM Proxy: {redact_and_sanitize(print_statement)}") # noqa: T201 def _get_email_logger_class(): diff --git a/litellm/utils.py b/litellm/utils.py index a11c5500503..0e41ae8df9c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -395,7 +395,7 @@ from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig -from ._logging import _is_debugging_on, verbose_logger +from ._logging import _is_debugging_on, redact_and_sanitize, verbose_logger from .caching.caching import ( AzureBlobCache, Cache, @@ -492,7 +492,7 @@ def print_verbose( elif log_level == "ERROR": verbose_logger.error(print_statement) if litellm.set_verbose is True and logger_only is False: - print(print_statement) # noqa: T201 + print(redact_and_sanitize(print_statement)) # noqa: T201 except Exception: pass diff --git a/tests/test_litellm/test_secret_redaction.py b/tests/test_litellm/test_secret_redaction.py index 7152eea7c9f..8393df98bdb 100644 --- a/tests/test_litellm/test_secret_redaction.py +++ b/tests/test_litellm/test_secret_redaction.py @@ -1,3 +1,5 @@ +import functools +import importlib import logging import sys from io import StringIO @@ -5,10 +7,13 @@ from unittest.mock import patch import pytest +import litellm from litellm._logging import ( JsonFormatter, _redact_string, _secret_filter, + redact_and_sanitize, + sanitize_control_characters, verbose_logger, verbose_proxy_logger, verbose_router_logger, @@ -381,3 +386,70 @@ def test_non_pem_private_key_value_redacted(): def test_normal_vertex_log_not_redacted(): msg = "Vertex: Loading vertex credentials, is_file_path=True, current dir /app" assert redact_string(msg) == msg + + +# ── set_verbose stdout paths (print_verbose) ── + + +def test_sanitize_control_characters_escapes_injection_vectors(): + raw = "line1\r\n2026-01-01 INFO forged\x1b[31mred\x1b[0m\u2028\u2029\x00 tab\there" + result = sanitize_control_characters(raw) + assert "\n" not in result + assert "\r" not in result + assert "\x1b" not in result + assert "\u2028" not in result + assert "\u2029" not in result + assert "\x00" not in result + assert "\\r\\n" in result + assert "\\x1b[31mred" in result + assert "\\u2028\\u2029\\x00" in result + assert "tab\there" in result + + +def test_redact_and_sanitize_masks_secrets_and_control_chars(): + result = redact_and_sanitize({"api_key": SECRET, "model": "gpt-5", "note": "a\nb"}) + assert SECRET not in result + assert "REDACTED" in result + assert "\n" not in result + assert "gpt-5" in result + + +_STDOUT_PRINT_VERBOSE_TARGETS = [ + ("litellm.utils", "print_verbose"), + ("litellm.main", "print_verbose"), + ("litellm.caching.caching", "print_verbose"), + ("litellm.litellm_core_utils.streaming_handler", "print_verbose"), + ("litellm.proxy.utils", "print_verbose"), + ("litellm.proxy.hooks.prompt_injection_detection", "_OPTIONAL_PromptInjectionDetection.print_verbose"), + ("litellm.proxy.hooks.batch_redis_get", "_PROXY_BatchRedisRequests.print_verbose"), + ("litellm.proxy.hooks.parallel_request_limiter", "_PROXY_MaxParallelRequestsHandler.print_verbose"), + ("litellm.proxy.guardrails.guardrail_hooks.presidio", "_OPTIONAL_PresidioPIIMasking.print_verbose"), +] + + +@pytest.mark.parametrize("module_path,attribute_path", _STDOUT_PRINT_VERBOSE_TARGETS) +def test_print_verbose_stdout_is_redacted_and_sanitized(module_path, attribute_path, capsys): + """Every set_verbose stdout path must mask secrets and escape control characters. + + The stdout branch bypasses the logging handlers, so SecretRedactionFilter + never sees it; an unpatched copy of print_verbose leaks the raw api_key and + lets model-controlled content forge log lines. + """ + module = importlib.import_module(module_path) + attribute_names = attribute_path.split(".") + print_verbose = functools.reduce(getattr, attribute_names, module) + payload = f"api_key={SECRET} chunk=gpt-5\r\n2026-01-01 INFO forged line \x1b[31mred\x1b[0m" + arguments = (None, payload) if len(attribute_names) > 1 else (payload,) + + with patch.object(litellm, "set_verbose", True): + print_verbose(*arguments) + + out = capsys.readouterr().out + assert out.strip(), f"{module_path} printed nothing with set_verbose=True" + assert SECRET not in out + assert "REDACTED" in out + assert "\r" not in out + assert "\x1b" not in out + assert out.count("\n") == 1 + assert "\\r\\n" in out + assert "\\x1b[31mred" in out