fix(logging): redact secrets and escape control chars on set_verbose stdout paths

This commit is contained in:
Devin AI 2026-07-26 12:11:19 +00:00
parent 24123269cc
commit ca6ddfe93e
16 changed files with 133 additions and 28 deletions

View file

@ -16,7 +16,7 @@ from litellm.proxy.guardrails._content_utils import (
iter_message_text,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm._logging import verbose_proxy_logger
from litellm._logging import redact_and_sanitize, verbose_proxy_logger
from fastapi import HTTPException
@ -54,7 +54,7 @@ class _ENTERPRISE_BannedKeywords(CustomLogger):
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose is True:
print(print_statement) # noqa
print(redact_and_sanitize(print_statement)) # noqa
def test_violation(self, test_str: str):
for word in self.banned_keywords_list:

View file

@ -13,7 +13,7 @@ from litellm.proxy.utils import PrismaClient
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth, LiteLLM_EndUserTable
from litellm.integrations.custom_logger import CustomLogger
from litellm._logging import verbose_proxy_logger
from litellm._logging import redact_and_sanitize, verbose_proxy_logger
from fastapi import HTTPException
@ -51,7 +51,7 @@ class _ENTERPRISE_BlockedUserList(CustomLogger):
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose is True:
print(print_statement) # noqa
print(redact_and_sanitize(print_statement)) # noqa
async def async_pre_call_hook(
self,

View file

@ -9,7 +9,7 @@
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._logging import redact_and_sanitize, verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails._content_utils import iter_message_text
@ -81,7 +81,7 @@ class _ENTERPRISE_GoogleTextModeration(CustomLogger):
try:
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose:
print(print_statement) # noqa
print(redact_and_sanitize(print_statement)) # noqa
except Exception:
pass

View file

@ -20,7 +20,7 @@ from typing import Literal, Optional
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._logging import redact_and_sanitize, verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.utils import CallTypesLiteral, Choices, ModelResponse
@ -55,7 +55,7 @@ class _ENTERPRISE_LlamaGuard(CustomLogger):
try:
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose:
print(print_statement) # noqa
print(redact_and_sanitize(print_statement)) # noqa
except Exception:
pass

View file

@ -14,7 +14,7 @@ import aiohttp
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._logging import redact_and_sanitize, verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
from litellm.secret_managers.main import get_secret_str
@ -42,7 +42,7 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
try:
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose:
print(print_statement) # noqa
print(redact_and_sanitize(print_statement)) # noqa
except Exception:
pass

View file

@ -1,6 +1,7 @@
import ast
import logging
import os
import re
import sys
from datetime import datetime
from logging import Formatter
@ -43,6 +44,37 @@ def redact_secrets(value: str) -> str:
return _redact_string(value)
_CONTROL_CHAR_RE = re.compile(r"[\x00-\x08\x0a-\x1f\x7f-\x9f\u2028\u2029]")
_CONTROL_CHAR_NAMES = {"\n": "\\n", "\r": "\\r", "\x0b": "\\v", "\x0c": "\\f", "\x08": "\\b", "\x07": "\\a"}
def _escape_control_character(match: "re.Match[str]") -> str:
char = match.group()
named = _CONTROL_CHAR_NAMES.get(char)
if named is not None:
return named
codepoint = ord(char)
return f"\\x{codepoint:02x}" if codepoint < 0x100 else f"\\u{codepoint:04x}"
def sanitize_control_characters(value: str) -> str:
"""Escape characters that let untrusted content forge log lines or drive the terminal.
Covers C0/C1 controls (CR, LF, ESC, ...) and the Unicode line/paragraph
separators; tab is left alone since it cannot start a new log record.
"""
return _CONTROL_CHAR_RE.sub(_escape_control_character, value)
def redact_and_sanitize(print_statement: object) -> str:
"""Render an arbitrary value for a stdout/print sink: secrets masked, control characters escaped.
Use this for the `litellm.set_verbose` print paths, which bypass the
logging handlers and therefore SecretRedactionFilter.
"""
return sanitize_control_characters(redact_secrets(str(print_statement)))
class SecretRedactionFilter(logging.Filter):
"""Scrubs known secret/credential patterns from log records."""
@ -413,7 +445,7 @@ def _enable_debugging():
def print_verbose(print_statement):
try:
if set_verbose:
print(redact_secrets(str(print_statement))) # noqa: T201
print(redact_and_sanitize(print_statement)) # noqa: T201
except Exception:
pass

View file

@ -18,7 +18,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
from pydantic import BaseModel
import litellm
from litellm._logging import verbose_logger
from litellm._logging import redact_and_sanitize, verbose_logger
from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
from litellm.types.caching import *
@ -41,7 +41,7 @@ def print_verbose(print_statement):
try:
verbose_logger.debug(print_statement)
if litellm.set_verbose:
print(print_statement) # noqa: T201
print(redact_and_sanitize(print_statement)) # noqa: T201
except Exception:
pass

View file

@ -26,6 +26,7 @@ from pydantic import BaseModel
import litellm
from litellm import verbose_logger
from litellm._logging import redact_and_sanitize
from litellm._uuid import uuid
from litellm.litellm_core_utils.model_response_utils import (
is_model_response_stream_empty,
@ -93,7 +94,7 @@ def is_async_iterable(obj: Any) -> bool:
def print_verbose(print_statement):
try:
if litellm.set_verbose:
print(print_statement) # noqa: T201
print(redact_and_sanitize(print_statement)) # noqa: T201
except Exception:
pass

View file

@ -41,7 +41,7 @@ from typing import (
get_args,
)
from litellm._logging import _redact_string
from litellm._logging import _redact_string, redact_and_sanitize
from litellm._uuid import uuid
if TYPE_CHECKING:
@ -8381,7 +8381,7 @@ def print_verbose(print_statement):
try:
verbose_logger.debug(print_statement)
if litellm.set_verbose:
print(print_statement) # noqa: T201
print(redact_and_sanitize(print_statement)) # noqa: T201
except Exception:
pass

View file

@ -30,7 +30,7 @@ import aiohttp
import litellm
from litellm import get_secret
from litellm._logging import verbose_proxy_logger
from litellm._logging import redact_and_sanitize, verbose_proxy_logger
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
@ -1320,7 +1320,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
try:
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose:
print(print_statement) # noqa: T201
print(redact_and_sanitize(print_statement)) # noqa: T201
except Exception:
pass

View file

@ -9,7 +9,7 @@ from typing import Literal, Optional
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._logging import redact_and_sanitize, verbose_proxy_logger
from litellm.caching.caching import DualCache, InMemoryCache, RedisCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy._types import UserAPIKeyAuth
@ -31,7 +31,7 @@ class _PROXY_BatchRedisRequests(CustomLogger):
elif debug_level == "INFO":
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose is True:
print(print_statement) # noqa: T201
print(redact_and_sanitize(print_statement)) # noqa: T201
async def async_pre_call_hook(
self,

View file

@ -8,7 +8,7 @@ from typing_extensions import TypedDict
import litellm
from litellm import DualCache, EmbeddingResponse, ModelResponse, TextCompletionResponse
from litellm._logging import verbose_proxy_logger
from litellm._logging import redact_and_sanitize, verbose_proxy_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
from litellm.proxy._types import CommonProxyErrors, CurrentItemRateLimit, UserAPIKeyAuth
@ -51,7 +51,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
try:
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose:
print(print_statement) # noqa: T201
print(redact_and_sanitize(print_statement)) # noqa: T201
except Exception:
pass

View file

@ -13,7 +13,7 @@ from typing import List, Literal, Optional
from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._logging import redact_and_sanitize, verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.constants import DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD
from litellm.integrations.custom_logger import CustomLogger
@ -72,7 +72,7 @@ class _OPTIONAL_PromptInjectionDetection(CustomLogger):
verbose_proxy_logger.debug(print_statement)
if litellm.set_verbose is True:
print(print_statement) # noqa: T201
print(redact_and_sanitize(print_statement)) # noqa: T201
def update_environment(self, router: Optional[Router] = None):
self.llm_router = router

View file

@ -88,7 +88,7 @@ from litellm import (
ModelResponseStream,
Router,
)
from litellm._logging import _redact_string, verbose_proxy_logger
from litellm._logging import _redact_string, redact_and_sanitize, verbose_proxy_logger
from litellm._service_logger import ServiceLogging, ServiceTypes
from litellm.caching.caching import DualCache, RedisCache
from litellm.caching.dual_cache import LimitedSizeOrderedDict
@ -196,7 +196,7 @@ def print_verbose(print_statement):
verbose_proxy_logger.debug("{}\n{}".format(print_statement, traceback.format_exc()))
if litellm.set_verbose:
print(f"LiteLLM Proxy: {_redact_string(str(print_statement))}") # noqa: T201
print(f"LiteLLM Proxy: {redact_and_sanitize(print_statement)}") # noqa: T201
def _get_email_logger_class():

View file

@ -395,7 +395,7 @@ from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig
from ._logging import _is_debugging_on, verbose_logger
from ._logging import _is_debugging_on, redact_and_sanitize, verbose_logger
from .caching.caching import (
AzureBlobCache,
Cache,
@ -492,7 +492,7 @@ def print_verbose(
elif log_level == "ERROR":
verbose_logger.error(print_statement)
if litellm.set_verbose is True and logger_only is False:
print(print_statement) # noqa: T201
print(redact_and_sanitize(print_statement)) # noqa: T201
except Exception:
pass

View file

@ -1,3 +1,5 @@
import functools
import importlib
import logging
import sys
from io import StringIO
@ -5,10 +7,13 @@ from unittest.mock import patch
import pytest
import litellm
from litellm._logging import (
JsonFormatter,
_redact_string,
_secret_filter,
redact_and_sanitize,
sanitize_control_characters,
verbose_logger,
verbose_proxy_logger,
verbose_router_logger,
@ -381,3 +386,70 @@ def test_non_pem_private_key_value_redacted():
def test_normal_vertex_log_not_redacted():
msg = "Vertex: Loading vertex credentials, is_file_path=True, current dir /app"
assert redact_string(msg) == msg
# ── set_verbose stdout paths (print_verbose) ──
def test_sanitize_control_characters_escapes_injection_vectors():
raw = "line1\r\n2026-01-01 INFO forged\x1b[31mred\x1b[0m\u2028\u2029\x00 tab\there"
result = sanitize_control_characters(raw)
assert "\n" not in result
assert "\r" not in result
assert "\x1b" not in result
assert "\u2028" not in result
assert "\u2029" not in result
assert "\x00" not in result
assert "\\r\\n" in result
assert "\\x1b[31mred" in result
assert "\\u2028\\u2029\\x00" in result
assert "tab\there" in result
def test_redact_and_sanitize_masks_secrets_and_control_chars():
result = redact_and_sanitize({"api_key": SECRET, "model": "gpt-5", "note": "a\nb"})
assert SECRET not in result
assert "REDACTED" in result
assert "\n" not in result
assert "gpt-5" in result
_STDOUT_PRINT_VERBOSE_TARGETS = [
("litellm.utils", "print_verbose"),
("litellm.main", "print_verbose"),
("litellm.caching.caching", "print_verbose"),
("litellm.litellm_core_utils.streaming_handler", "print_verbose"),
("litellm.proxy.utils", "print_verbose"),
("litellm.proxy.hooks.prompt_injection_detection", "_OPTIONAL_PromptInjectionDetection.print_verbose"),
("litellm.proxy.hooks.batch_redis_get", "_PROXY_BatchRedisRequests.print_verbose"),
("litellm.proxy.hooks.parallel_request_limiter", "_PROXY_MaxParallelRequestsHandler.print_verbose"),
("litellm.proxy.guardrails.guardrail_hooks.presidio", "_OPTIONAL_PresidioPIIMasking.print_verbose"),
]
@pytest.mark.parametrize("module_path,attribute_path", _STDOUT_PRINT_VERBOSE_TARGETS)
def test_print_verbose_stdout_is_redacted_and_sanitized(module_path, attribute_path, capsys):
"""Every set_verbose stdout path must mask secrets and escape control characters.
The stdout branch bypasses the logging handlers, so SecretRedactionFilter
never sees it; an unpatched copy of print_verbose leaks the raw api_key and
lets model-controlled content forge log lines.
"""
module = importlib.import_module(module_path)
attribute_names = attribute_path.split(".")
print_verbose = functools.reduce(getattr, attribute_names, module)
payload = f"api_key={SECRET} chunk=gpt-5\r\n2026-01-01 INFO forged line \x1b[31mred\x1b[0m"
arguments = (None, payload) if len(attribute_names) > 1 else (payload,)
with patch.object(litellm, "set_verbose", True):
print_verbose(*arguments)
out = capsys.readouterr().out
assert out.strip(), f"{module_path} printed nothing with set_verbose=True"
assert SECRET not in out
assert "REDACTED" in out
assert "\r" not in out
assert "\x1b" not in out
assert out.count("\n") == 1
assert "\\r\\n" in out
assert "\\x1b[31mred" in out